tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[BugFix] Fix view based rewrite bugs (backport #62918) (#63013) 

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
Co-authored-by: shuming.li <ming.moriarty@gmail.com>
This commit is contained in:
mergify[bot] 2025-09-12 14:03:41 +08:00 committed by GitHub
parent 055bf5a488
commit fcb46895e3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 462 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java
View File

@ -25,7 +25,6 @@ import com.starrocks.catalog.system.SystemTable;
import com.starrocks.connector.metadata.MetadataTable;
import com.starrocks.qe.ConnectContext;
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.sql.StatementPlanner;
import com.starrocks.sql.analyzer.Analyzer;
import com.starrocks.sql.analyzer.Authorizer;
import com.starrocks.sql.ast.AstTraverser;
@ -107,7 +106,7 @@ public class ColumnPrivilege {
*/
ColumnRefFactory columnRefFactory = new ColumnRefFactory();
LogicalPlan logicalPlan;
MVTransformerContext mvTransformerContext = StatementPlanner.makeMVTransformerContext(context.getSessionVariable());
MVTransformerContext mvTransformerContext = MVTransformerContext.of(context, true);
TransformerContext transformerContext = new TransformerContext(columnRefFactory, context, mvTransformerContext);
logicalPlan = new RelationTransformer(transformerContext).transformWithSelectLimit(stmt.getQueryRelation());

4
fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java
View File

@ -200,8 +200,8 @@ public class PartitionBasedMvRefreshProcessor extends BaseTaskRunProcessor {
Tracers.register(context.getCtx());
QueryDebugOptions queryDebugOptions = context.getCtx().getSessionVariable().getQueryDebugOptions();
// init to collect the base timer for refresh profile
Tracers.Mode mvRefreshTraceMode = queryDebugOptions.getMvRefreshTraceMode();
Tracers.Module mvRefreshTraceModule = queryDebugOptions.getMvRefreshTraceModule();
Tracers.Mode mvRefreshTraceMode = queryDebugOptions.getTraceMode();
Tracers.Module mvRefreshTraceModule = queryDebugOptions.getTraceModule();
Tracers.init(mvRefreshTraceMode, mvRefreshTraceModule, true, false);
IMaterializedViewMetricsEntity mvEntity = null;

15
fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
View File

@ -39,7 +39,6 @@ import com.starrocks.http.HttpConnectContext;
import com.starrocks.planner.PlanFragment;
import com.starrocks.planner.ResultSink;
import com.starrocks.qe.ConnectContext;
import com.starrocks.qe.SessionVariable;
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.service.FrontendOptions;
import com.starrocks.service.arrow.flight.sql.ArrowFlightSqlConnectContext;
@ -277,16 +276,6 @@ public class StatementPlanner {
return areTablesCopySafe && !session.getSessionVariable().isCboUseDBLock();
}
/**
* Create a map from opt expression to parse node for the optimizer to use which only used in text match rewrite for mv.
*/
public static MVTransformerContext makeMVTransformerContext(SessionVariable sessionVariable) {
if (sessionVariable.isEnableMaterializedViewTextMatchRewrite()) {
return new MVTransformerContext();
}
return null;
}
private static ExecPlan createQueryPlan(StatementBase stmt,
ConnectContext session,
TResultSinkType resultSinkType) {
@ -296,7 +285,7 @@ public class StatementPlanner {
// 1. Build Logical plan
ColumnRefFactory columnRefFactory = new ColumnRefFactory();
LogicalPlan logicalPlan;
MVTransformerContext mvTransformerContext = makeMVTransformerContext(session.getSessionVariable());
MVTransformerContext mvTransformerContext = MVTransformerContext.of(session, true);
try (Timer ignored = Tracers.watchScope("Transformer")) {
// get a logicalPlan without inlining views
@ -363,7 +352,7 @@ public class StatementPlanner {
}
LogicalPlan logicalPlan;
MVTransformerContext mvTransformerContext = makeMVTransformerContext(session.getSessionVariable());
MVTransformerContext mvTransformerContext = MVTransformerContext.of(session, true);
try (Timer ignored = Tracers.watchScope("Transformer")) {
// get a logicalPlan without inlining views
TransformerContext transformerContext = new TransformerContext(columnRefFactory, session, mvTransformerContext);

24
fe/fe-core/src/main/java/com/starrocks/sql/common/QueryDebugOptions.java
View File

@ -38,12 +38,20 @@ public class QueryDebugOptions {
@SerializedName(value = "enableQueryTraceLog")
private boolean enableQueryTraceLog = false;
@Deprecated
@SerializedName(value = "mvRefreshTraceMode")
private String mvRefreshTraceMode;
@Deprecated
@SerializedName(value = "mvRefreshTraceModule")
private String mvRefreshTraceModule;
@SerializedName(value = "traceMode")
private String traceMode;
@SerializedName(value = "traceModule")
private String traceModule;
public static class ExecDebugOption {
@SerializedName(value = "plan_node_id")
private int planNodeId;
@ -93,12 +101,20 @@ public class QueryDebugOptions {
this.enableQueryTraceLog = enableQueryTraceLog;
}
public Tracers.Mode getMvRefreshTraceMode() {
return Strings.isEmpty(mvRefreshTraceMode) ? Tracers.Mode.TIMER : Tracers.Mode.valueOf(mvRefreshTraceMode);
public Tracers.Mode getTraceMode() {
return Strings.isEmpty(traceMode) ? Tracers.Mode.TIMER : Tracers.Mode.valueOf(traceMode.toUpperCase());
}
public Tracers.Module getMvRefreshTraceModule() {
return Strings.isEmpty(mvRefreshTraceModule) ? Tracers.Module.BASE : Tracers.Module.valueOf(mvRefreshTraceModule);
public Tracers.Module getTraceModule() {
return Strings.isEmpty(traceModule) ? Tracers.Module.BASE : Tracers.Module.valueOf(traceModule.toUpperCase());
}
public void setTraceMode(String traceMode) {
this.traceMode = traceMode;
}
public void setTraceModule(String traceModule) {
this.traceModule = traceModule;
}
public List<ExecDebugOption> getExecDebugOptions() {

7
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java
View File

@ -25,6 +25,7 @@ import com.starrocks.sql.optimizer.base.ColumnRefFactory;
import com.starrocks.sql.optimizer.rule.RuleType;
import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils;
import com.starrocks.sql.optimizer.transformer.LogicalPlan;
import com.starrocks.sql.optimizer.transformer.MVTransformerContext;
public class MaterializedViewOptimizer {
public MvPlanContext optimize(MaterializedView mv,
@ -108,11 +109,11 @@ public class MaterializedViewOptimizer {
connectContext.getSessionVariable().setSemiJoinDeduplicateMode(-1);
}
MVTransformerContext mvTransformerContext = new MVTransformerContext(connectContext, inlineView);
try {
// get optimized plan of mv's defined query
Pair<OptExpression, LogicalPlan> plans =
MvUtils.getRuleOptimizedLogicalPlan(stmt, columnRefFactory, connectContext, optimizerOptions,
inlineView);
Pair<OptExpression, LogicalPlan> plans = MvUtils.getRuleOptimizedLogicalPlan(stmt, columnRefFactory,
connectContext, optimizerOptions, mvTransformerContext);
if (plans == null) {
return new MvPlanContext(false, "No query plan for it");
}

14
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java
View File

@ -274,6 +274,10 @@ public class MvRewritePreprocessor {
// means there is no plan with view
return;
}
Set<String> viewNames = logicalViewScanOperators.stream()
.map(op -> op.getTable().getName()).collect(Collectors.toSet());
logMVPrepare(connectContext, "[ViewBasedRewrite] There are {} view scan operators in the query plan",
viewNames);
// optimize logical plan with view
OptExpression optViewScanExpressions = MvUtils.optimizeViewPlan(
logicalPlanWithViewInline, connectContext, requiredColumns, columnRefFactory);
@ -306,9 +310,13 @@ public class MvRewritePreprocessor {
// add a projection to make predicate push-down rules work.
Projection projection = viewScanOperator.getProjection();
LogicalProjectOperator projectOperator = new LogicalProjectOperator(projection.getColumnRefMap());
OptExpression projectionExpr = OptExpression.create(projectOperator, viewScanExpr);
return projectionExpr;
if (projection != null) {
LogicalProjectOperator projectOperator = new LogicalProjectOperator(projection.getColumnRefMap());
OptExpression projectionExpr = OptExpression.create(projectOperator, viewScanExpr);
return projectionExpr;
} else {
return viewScanExpr;
}
} else {
for (OptExpression input : logicalTree.getInputs()) {
OptExpression newInput = extractLogicalPlanWithView(input, viewScans);

4
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/SubqueryUtils.java
View File

@ -44,6 +44,7 @@ import com.starrocks.sql.optimizer.rewrite.scalar.ScalarOperatorRewriteRule;
import com.starrocks.sql.optimizer.transformer.CTETransformerContext;
import com.starrocks.sql.optimizer.transformer.ExpressionMapping;
import com.starrocks.sql.optimizer.transformer.LogicalPlan;
import com.starrocks.sql.optimizer.transformer.MVTransformerContext;
import com.starrocks.sql.optimizer.transformer.OptExprBuilder;
import com.starrocks.sql.optimizer.transformer.RelationTransformer;
@ -103,7 +104,8 @@ public class SubqueryUtils {
relation.getOrderBy().clear();
}
return new RelationTransformer(columnRefFactory, session, outer, cteContext).transform(relation);
return new RelationTransformer(columnRefFactory, session, outer, cteContext,
new MVTransformerContext(session, true)).transform(relation);
}
private static Function getAggregateFunction(String functionName, Type[] argTypes) {

49
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Utils.java
View File

@ -888,22 +888,45 @@ public class Utils {
if (newProjectionMap == null || newProjectionMap.isEmpty()) {
return input;
}
Operator newOp = input.getOp();
if (newOp.getProjection() == null || newOp.getProjection().getColumnRefMap().isEmpty()) {
newOp.setProjection(new Projection(newProjectionMap));
} else {
// merge two projections
ReplaceColumnRefRewriter rewriter = new ReplaceColumnRefRewriter(newOp.getProjection().getColumnRefMap());
Map<ColumnRefOperator, ScalarOperator> resultMap = Maps.newHashMap();
for (Map.Entry<ColumnRefOperator, ScalarOperator> entry : newProjectionMap.entrySet()) {
ScalarOperator result = rewriter.rewrite(entry.getValue());
resultMap.put(entry.getKey(), result);
}
newOp.setProjection(new Projection(resultMap));
}
Operator inputOp = input.getOp();
// merge two projections
Projection newProjection = new Projection(mergeWithProject(newProjectionMap, inputOp.getProjection()));
inputOp.setProjection(newProjection);
return input;
}
/**
* Merge projection1 -> projection2, use projection1's output as the final output.
*/
public static Projection mergeWithProject(Projection projection1,
Projection projection2) {
if (projection1 == null || projection1.getColumnRefMap() == null) {
return projection1;
}
return new Projection(mergeWithProject(projection1.getColumnRefMap(), projection2));
}
/**
* Merge input mapping with projection's mapping, return a new mapping based on the existed projection.
*/
public static Map<ColumnRefOperator, ScalarOperator> mergeWithProject(Map<ColumnRefOperator, ScalarOperator> input,
Projection projection) {
if (input == null || input.isEmpty()) {
return input;
}
if (projection == null || projection.getColumnRefMap() == null) {
return input;
}
ReplaceColumnRefRewriter rewriter = new ReplaceColumnRefRewriter(projection.getColumnRefMap());
Map<ColumnRefOperator, ScalarOperator> resultMap = Maps.newHashMap();
for (Map.Entry<ColumnRefOperator, ScalarOperator> entry : input.entrySet()) {
ScalarOperator result = rewriter.rewrite(entry.getValue());
resultMap.put(entry.getKey(), result);
}
return resultMap;
}
/**
* Check if the optExpression has applied the rule in recursively
* @param optExpression input optExpression to be checked

35
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java
View File

@ -109,6 +109,7 @@ import com.starrocks.sql.optimizer.rule.RuleType;
import com.starrocks.sql.optimizer.rule.mv.MVUtils;
import com.starrocks.sql.optimizer.rule.mv.MaterializedViewWrapper;
import com.starrocks.sql.optimizer.transformer.LogicalPlan;
import com.starrocks.sql.optimizer.transformer.MVTransformerContext;
import com.starrocks.sql.optimizer.transformer.RelationTransformer;
import com.starrocks.sql.optimizer.transformer.SqlToScalarOperatorTranslator;
import com.starrocks.sql.optimizer.transformer.TransformerContext;
@ -482,12 +483,12 @@ public class MvUtils {
ColumnRefFactory columnRefFactory,
ConnectContext connectContext,
OptimizerOptions optimizerOptions,
boolean inlineView) {
MVTransformerContext mvTransformerContext) {
Preconditions.checkState(mvStmt instanceof QueryStatement);
Analyzer.analyze(mvStmt, connectContext);
QueryRelation query = ((QueryStatement) mvStmt).getQueryRelation();
TransformerContext transformerContext =
new TransformerContext(columnRefFactory, connectContext, inlineView, null);
new TransformerContext(columnRefFactory, connectContext, mvTransformerContext);
LogicalPlan logicalPlan = new RelationTransformer(transformerContext).transform(query);
Optimizer optimizer =
OptimizerFactory.create(
@ -1218,40 +1219,40 @@ public class MvUtils {
LogicalViewScanOperator viewScanOperator = op.cast();
OptExpression inlineViewPlan = viewScanOperator.getOriginalPlanEvaluator();
if (viewScanOperator.getPredicate() != null) {
Operator inlineViewOp = inlineViewPlan.getOp();
// original map records inlined view's column ref to non-inlined view's column ref mapping,
// now we need to rewrite non-inlined view's column ref to inlined view's column ref,
// so we need to reverse the mapping.
Map<ColumnRefOperator, ScalarOperator> originalColumnRefToInlinedColumnRefMap =
Maps.newHashMap(viewScanOperator.getOriginalColumnRefToInlinedColumnRefMap());
// add new added column ref mapping
if (viewScanOperator.getProjection() != null) {
viewScanOperator.getProjection().getColumnRefMap()
.entrySet()
.stream()
.forEach(e -> originalColumnRefToInlinedColumnRefMap.put(e.getKey(), e.getValue()));
}
Map<ColumnRefOperator, ScalarOperator> reverseColumnRefMap = originalColumnRefToInlinedColumnRefMap
.entrySet()
.stream()
.collect(Collectors.toMap(e -> (ColumnRefOperator) e.getValue(), e -> e.getKey()));
// If viewScanOperator contains predicate, we need to rewrite them, otherwise predicate will be lost.
if (inlineViewOp.getProjection() != null) {
// if inline view's projection is not null, also need to rewrite inline view's projection
reverseColumnRefMap = Utils.mergeWithProject(reverseColumnRefMap, inlineViewOp.getProjection());
}
ReplaceColumnRefRewriter rewriter = new ReplaceColumnRefRewriter(reverseColumnRefMap);
Operator.Builder builder = OperatorBuilderFactory.build(inlineViewPlan.getOp());
builder.withOperator(inlineViewPlan.getOp());
Operator.Builder builder = OperatorBuilderFactory.build(inlineViewOp);
builder.withOperator(inlineViewOp);
if (viewScanOperator.getPredicate() != null) {
// rewrite predicate
// If viewScanOperator contains predicate, we need to rewrite them, otherwise predicate will be lost.
ScalarOperator rewrittenPredicate = rewriter.rewrite(viewScanOperator.getPredicate());
builder.setPredicate(rewrittenPredicate);
}
// rewrite projection
if (viewScanOperator.getPredicate() != null) {
Map<ColumnRefOperator, ScalarOperator> newColumnRefMap = viewScanOperator
if (viewScanOperator.getProjection() != null) {
// merge inline view's projection with view scan's projection
Map<ColumnRefOperator, ScalarOperator> newColumnRefMap = Maps.newHashMap();
viewScanOperator
.getProjection().getColumnRefMap()
.entrySet()
.stream()
.map(e -> Maps.immutableEntry(e.getKey(), rewriter.rewrite(e.getValue())))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
.forEach(e -> newColumnRefMap.put(e.getKey(), e.getValue()));
builder.setProjection(new Projection(newColumnRefMap));
}
Operator newInlineViewPlanOp = builder.build();

53
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/MVTransformerContext.java
View File

@ -15,6 +15,9 @@ package com.starrocks.sql.optimizer.transformer;
import com.google.common.collect.Maps;
import com.starrocks.analysis.ParseNode;
import com.starrocks.catalog.View;
import com.starrocks.qe.ConnectContext;
import com.starrocks.qe.SessionVariable;
import com.starrocks.sql.optimizer.operator.Operator;
import com.starrocks.sql.util.Box;
@ -26,7 +29,53 @@ public class MVTransformerContext {
// if `Operator`'s equals method is true.
private final Map<Box<Operator>, ParseNode> opToASTMap = Maps.newHashMap();
public MVTransformerContext() {
private final ConnectContext connectContext;
// Whether the current transformer is for inline view, in some cases(eg: mv optimizer builder), needs to disable inline
// view directly, otherwise it's true by default.
private final boolean isInlineView;
// Whether enable text based mv rewrite
private final boolean isEnableTextBasedMVRewrite;
// Whether enable view based mv rewrite
private final boolean isEnableViewBasedMVRewrite;
public MVTransformerContext(ConnectContext context, boolean isInlineView) {
this.connectContext = context;
this.isInlineView = isInlineView;
// set session variable
SessionVariable sessionVariable = context.getSessionVariable();
if (sessionVariable.isDisableMaterializedViewRewrite() ||
!sessionVariable.isEnableMaterializedViewRewrite()) {
this.isEnableTextBasedMVRewrite = false;
this.isEnableViewBasedMVRewrite = false;
} else {
this.isEnableTextBasedMVRewrite = sessionVariable.isEnableMaterializedViewTextMatchRewrite();
this.isEnableViewBasedMVRewrite = sessionVariable.isEnableViewBasedMvRewrite();
}
}
public static MVTransformerContext of(ConnectContext context, boolean isInlineView) {
return new MVTransformerContext(context, isInlineView);
}
/**
* Whether enable view based mv rewrite, only if
* - session variable enable_view_based_mv_rewrite is true
* - view contains related mvs
*/
public boolean isEnableViewBasedMVRewrite(View view) {
if (view == null) {
return false;
}
// ensure view's related views not empty
return isEnableViewBasedMVRewrite && !view.getRelatedMaterializedViews().isEmpty();
}
public boolean isInlineView() {
return isInlineView;
}
public boolean isEnableTextBasedMVRewrite() {
return isEnableTextBasedMVRewrite;
}
/**
@ -35,7 +84,7 @@ public class MVTransformerContext {
* @param ast AST tree of this operator
*/
public void registerOpAST(Operator op, ParseNode ast) {
if (op == null) {
if (op == null || !isEnableTextBasedMVRewrite) {
return;
}
opToASTMap.put(Box.of(op), ast);

6
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/QueryTransformer.java
View File

@ -67,18 +67,16 @@ public class QueryTransformer {
private final ConnectContext session;
private final List<ColumnRefOperator> correlation = new ArrayList<>();
private final CTETransformerContext cteContext;
private final boolean inlineView;
private final MVTransformerContext mvTransformerContext;
public static final String GROUPING_ID = "GROUPING_ID";
public static final String GROUPING = "GROUPING";
public QueryTransformer(ColumnRefFactory columnRefFactory, ConnectContext session,
CTETransformerContext cteContext, boolean inlineView,
CTETransformerContext cteContext,
MVTransformerContext mvTransformerContext) {
this.columnRefFactory = columnRefFactory;
this.session = session;
this.cteContext = cteContext;
this.inlineView = inlineView;
this.mvTransformerContext = mvTransformerContext;
}
@ -168,7 +166,7 @@ public class QueryTransformer {
private OptExprBuilder planFrom(Relation node, CTETransformerContext cteContext) {
TransformerContext transformerContext = new TransformerContext(
columnRefFactory, session, new ExpressionMapping(new Scope(RelationId.anonymous(), new RelationFields())),
cteContext, inlineView, mvTransformerContext);
cteContext, mvTransformerContext);
return new RelationTransformer(transformerContext).visit(node).getRootBuilder();
}

48
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/RelationTransformer.java
View File

@ -38,6 +38,7 @@ import com.starrocks.catalog.OlapTable;
import com.starrocks.catalog.Table;
import com.starrocks.catalog.TableFunction;
import com.starrocks.catalog.Type;
import com.starrocks.catalog.View;
import com.starrocks.common.Pair;
import com.starrocks.connector.ConnectorTableVersion;
import com.starrocks.connector.PointerType;
@ -167,19 +168,18 @@ public class RelationTransformer implements AstVisitor<LogicalPlan, ExpressionMa
private final ExpressionMapping outer;
private final CTETransformerContext cteContext;
private final List<ColumnRefOperator> correlation = new ArrayList<>();
private final boolean inlineView;
private final boolean enableViewBasedMvRewrite;
private final MVTransformerContext mvTransformerContext;
public RelationTransformer(ColumnRefFactory columnRefFactory, ConnectContext session) {
this(columnRefFactory, session,
new ExpressionMapping(new Scope(RelationId.anonymous(), new RelationFields())),
new CTETransformerContext(session.getSessionVariable().getCboCTEMaxLimit()));
new CTETransformerContext(session.getSessionVariable().getCboCTEMaxLimit()),
new MVTransformerContext(session, true));
}
public RelationTransformer(ColumnRefFactory columnRefFactory, ConnectContext session, ExpressionMapping outer,
CTETransformerContext cteContext) {
this(new TransformerContext(columnRefFactory, session, outer, cteContext, null));
CTETransformerContext cteContext, MVTransformerContext mvTransformerContext) {
this(new TransformerContext(columnRefFactory, session, outer, cteContext, mvTransformerContext));
}
public RelationTransformer(TransformerContext context) {
@ -187,8 +187,6 @@ public class RelationTransformer implements AstVisitor<LogicalPlan, ExpressionMa
this.session = context.getSession();
this.outer = context.getOuter();
this.cteContext = context.getCteContext();
this.inlineView = context.isInlineView();
this.enableViewBasedMvRewrite = context.isEnableViewBasedMvRewrite();
this.mvTransformerContext = context.getMVTransformerContext();
}
@ -245,7 +243,7 @@ public class RelationTransformer implements AstVisitor<LogicalPlan, ExpressionMa
LogicalPlan producerPlan =
new RelationTransformer(columnRefFactory, session,
new ExpressionMapping(new Scope(RelationId.anonymous(), new RelationFields())),
cteContext).transform(cteRelation.getCteQueryStatement().getQueryRelation());
cteContext, mvTransformerContext).transform(cteRelation.getCteQueryStatement().getQueryRelation());
OptExprBuilder produceOptBuilder =
new OptExprBuilder(produceOperator, Lists.newArrayList(producerPlan.getRootBuilder()),
producerPlan.getRootBuilder().getExpressionMapping());
@ -288,7 +286,7 @@ public class RelationTransformer implements AstVisitor<LogicalPlan, ExpressionMa
@Override
public LogicalPlan visitSelect(SelectRelation node, ExpressionMapping context) {
QueryTransformer queryTransformer = new QueryTransformer(columnRefFactory, session, cteContext,
inlineView, mvTransformerContext);
mvTransformerContext);
LogicalPlan logicalPlan = queryTransformer.plan(node, outer);
return logicalPlan;
}
@ -872,30 +870,44 @@ public class RelationTransformer implements AstVisitor<LogicalPlan, ExpressionMa
@Override
public LogicalPlan visitView(ViewRelation node, ExpressionMapping context) {
LogicalPlan logicalPlan = transform(node.getQueryStatement().getQueryRelation());
List<ColumnRefOperator> newOutputColumns = inlineView ? null : Lists.newArrayList();
if (inlineView) {
boolean isInlineView = isInlineView();
boolean isEnableViewBasedRewrite = isEnableViewBasedRewrite(node.getView());
if (isInlineView) {
// expand views in logical plan
OptExprBuilder builder = new OptExprBuilder(
logicalPlan.getRoot().getOp(),
logicalPlan.getRootBuilder().getInputs(),
new ExpressionMapping(node.getScope(), logicalPlan.getOutputColumn(), logicalPlan.getRootBuilder()
.getColumnRefToConstOperators()));
if (enableViewBasedMvRewrite) {
LogicalViewScanOperator viewScanOperator = buildViewScan(logicalPlan, node, newOutputColumns);
if (isEnableViewBasedRewrite) {
List<ColumnRefOperator> newOutputColumns = Lists.newArrayList();
LogicalViewScanOperator viewScanOperator = buildViewScan(logicalPlan, node, newOutputColumns, true);
builder.getRoot().getOp().setEquivalentOp(viewScanOperator);
}
return new LogicalPlan(builder, logicalPlan.getOutputColumn(), logicalPlan.getCorrelation());
} else {
// do not expand views in logical plan
LogicalViewScanOperator viewScanOperator = buildViewScan(logicalPlan, node, newOutputColumns);
List<ColumnRefOperator> newOutputColumns = Lists.newArrayList();
LogicalViewScanOperator viewScanOperator = buildViewScan(logicalPlan, node, newOutputColumns, false);
OptExprBuilder scanBuilder = new OptExprBuilder(viewScanOperator, Collections.emptyList(),
new ExpressionMapping(node.getScope(), newOutputColumns));
return new LogicalPlan(scanBuilder, newOutputColumns, List.of());
}
}
private LogicalViewScanOperator buildViewScan(
LogicalPlan logicalPlan, ViewRelation node, List<ColumnRefOperator> outputVariables) {
private boolean isInlineView() {
return mvTransformerContext != null ? mvTransformerContext.isInlineView() : true;
}
private boolean isEnableViewBasedRewrite(View view) {
return mvTransformerContext != null ? mvTransformerContext.isEnableViewBasedMVRewrite(view) : false;
}
private LogicalViewScanOperator buildViewScan(LogicalPlan logicalPlan,
ViewRelation node,
List<ColumnRefOperator> outputVariables,
boolean isInlineView) {
ImmutableMap.Builder<ColumnRefOperator, Column> colRefToColumnMetaMapBuilder = ImmutableMap.builder();
ImmutableMap.Builder<Column, ColumnRefOperator> columnMetaToColRefMapBuilder = ImmutableMap.builder();
@ -947,7 +959,7 @@ public class RelationTransformer implements AstVisitor<LogicalPlan, ExpressionMa
LogicalViewScanOperator scanOperator = new LogicalViewScanOperator(relationId,
node.getView(), columnRefOperatorToColumn, columnMetaToColRefMap,
new ColumnRefSet(logicalPlan.getOutputColumn()), newExprMapping, projectionMap);
if (inlineView) {
if (isInlineView) {
// add a projection to make sure output columns keep the same,
// because LogicalViewScanOperator should be logically equivalent to logicalPlan
Projection projection = new Projection(projectionMap);
@ -1138,7 +1150,7 @@ public class RelationTransformer implements AstVisitor<LogicalPlan, ExpressionMa
List<Expr> groupKeys = node.getGroupByKeys();
List<FunctionCallExpr> aggFunctions = node.getRewrittenAggFunctions();
QueryTransformer queryTransformer = new QueryTransformer(columnRefFactory, session, cteContext,
inlineView, mvTransformerContext);
mvTransformerContext);
OptExprBuilder builder = queryTransformer.aggregate(
queryPlan.getRootBuilder(), groupKeys, aggFunctions, null, ImmutableList.of());

37
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/TransformerContext.java
View File

@ -26,11 +26,6 @@ public class TransformerContext {
private final ExpressionMapping outer;
private final CTETransformerContext cteContext;
// whether to expand view in logical plan
// the origin strategy is true, means will inline view by default.
private final boolean inlineView;
private final boolean enableViewBasedMvRewrite;
private final MVTransformerContext mvTransformerContext;
public TransformerContext(
@ -39,17 +34,7 @@ public class TransformerContext {
MVTransformerContext mvTransformerContext) {
this(columnRefFactory, session,
new ExpressionMapping(new Scope(RelationId.anonymous(), new RelationFields())),
new CTETransformerContext(session.getSessionVariable().getCboCTEMaxLimit()), true, mvTransformerContext);
}
public TransformerContext(
ColumnRefFactory columnRefFactory,
ConnectContext session,
boolean inlineView,
MVTransformerContext mvTransformerContext) {
this(columnRefFactory, session,
new ExpressionMapping(new Scope(RelationId.anonymous(), new RelationFields())),
new CTETransformerContext(session.getSessionVariable().getCboCTEMaxLimit()), inlineView, mvTransformerContext);
new CTETransformerContext(session.getSessionVariable().getCboCTEMaxLimit()), mvTransformerContext);
}
public TransformerContext(
@ -58,22 +43,10 @@ public class TransformerContext {
ExpressionMapping outer,
CTETransformerContext cteContext,
MVTransformerContext mvTransformerContext) {
this(columnRefFactory, session, outer, cteContext, true, mvTransformerContext);
}
public TransformerContext(
ColumnRefFactory columnRefFactory,
ConnectContext session,
ExpressionMapping outer,
CTETransformerContext cteContext,
boolean inlineView,
MVTransformerContext mvTransformerContext) {
this.columnRefFactory = columnRefFactory;
this.session = session;
this.outer = outer;
this.cteContext = cteContext;
this.inlineView = inlineView;
this.enableViewBasedMvRewrite = session.getSessionVariable().isEnableViewBasedMvRewrite();
this.mvTransformerContext = mvTransformerContext;
}
@ -93,14 +66,6 @@ public class TransformerContext {
return cteContext;
}
public boolean isInlineView() {
return inlineView;
}
public boolean isEnableViewBasedMvRewrite() {
return enableViewBasedMvRewrite;
}
public MVTransformerContext getMVTransformerContext() {
return mvTransformerContext;
}

62
fe/fe-core/src/test/java/com/starrocks/sql/optimizer/UtilsTest.java
View File

@ -31,6 +31,7 @@ import com.starrocks.common.FeConstants;
import com.starrocks.qe.ConnectContext;
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.sql.ast.CreateDbStmt;
import com.starrocks.sql.optimizer.operator.Projection;
import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
import com.starrocks.sql.optimizer.operator.logical.LogicalPaimonScanOperator;
@ -386,4 +387,65 @@ public class UtilsTest {
Assertions.assertTrue(out / 2 < i);
}
}
@Test
public void testMergeWithProject1() {
Map<ColumnRefOperator, ScalarOperator> columnRefMap1 = new HashMap<>();
Map<ColumnRefOperator, ScalarOperator> columnRefMap2 = new HashMap<>();
for (int i = 0; i < 10; i++) {
ColumnRefOperator midColRef;
{
String colName = "v" + (i + 1);
ColumnRefOperator colRef = new ColumnRefOperator(i, Type.BIGINT, colName, true);
midColRef = new ColumnRefOperator(i + 50, Type.BIGINT, colName, true);
columnRefMap1.put(colRef, midColRef);
}
{
String colName = "v" + (i + 100);
ColumnRefOperator colRef = new ColumnRefOperator(i, Type.BIGINT, colName, true);
columnRefMap2.put(midColRef, colRef);
}
}
Projection projection1 = new Projection(columnRefMap1);
Projection projection2 = new Projection(columnRefMap2);
Projection mergedProjection = Utils.mergeWithProject(projection1, projection2);
Assertions.assertEquals(10, mergedProjection.getColumnRefMap().size());
for (int i = 0; i < 10; i++) {
ColumnRefOperator colRef = new ColumnRefOperator(i, Type.BIGINT, "v" + (i + 1), true);
Assertions.assertTrue(mergedProjection.getColumnRefMap().containsKey(colRef));
ScalarOperator scalarOperator = mergedProjection.getColumnRefMap().get(colRef);
Assertions.assertTrue(scalarOperator instanceof ColumnRefOperator);
Assertions.assertEquals("v" + (i + 100), ((ColumnRefOperator) scalarOperator).getName());
}
}
@Test
public void testMergeWithProject2() {
Map<ColumnRefOperator, ScalarOperator> columnRefMap1 = new HashMap<>();
Map<ColumnRefOperator, ScalarOperator> columnRefMap2 = new HashMap<>();
for (int i = 0; i < 10; i++) {
ColumnRefOperator midColRef;
{
String colName = "v" + (i + 1);
ColumnRefOperator colRef = new ColumnRefOperator(i, Type.BIGINT, colName, true);
midColRef = new ColumnRefOperator(i + 50, Type.BIGINT, colName, true);
columnRefMap1.put(colRef, midColRef);
}
{
String colName = "v" + (i + 100);
ColumnRefOperator colRef = new ColumnRefOperator(i, Type.BIGINT, colName, true);
columnRefMap2.put(midColRef, colRef);
}
}
Projection projection2 = new Projection(columnRefMap2);
Map<ColumnRefOperator, ScalarOperator> mergedColRefMap = Utils.mergeWithProject(columnRefMap1, projection2);
Assertions.assertEquals(10, mergedColRefMap.size());
for (int i = 0; i < 10; i++) {
ColumnRefOperator colRef = new ColumnRefOperator(i, Type.BIGINT, "v" + (i + 1), true);
Assertions.assertTrue(mergedColRefMap.containsKey(colRef));
ScalarOperator scalarOperator = mergedColRefMap.get(colRef);
Assertions.assertTrue(scalarOperator instanceof ColumnRefOperator);
Assertions.assertEquals("v" + (i + 100), ((ColumnRefOperator) scalarOperator).getName());
}
}
}

7
fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java
View File

@ -30,6 +30,7 @@ import com.starrocks.catalog.MvPlanContext;
import com.starrocks.catalog.MvRefreshArbiter;
import com.starrocks.catalog.MvUpdateInfo;
import com.starrocks.catalog.Table;
import com.starrocks.catalog.View;
import com.starrocks.common.DdlException;
import com.starrocks.common.Pair;
import com.starrocks.common.util.RuntimeProfile;
@ -677,4 +678,10 @@ public class MVTestBase extends StarRocksTestBase {
addListPartition(tbl, pName, val);
}
}
public View getView(String viewName) {
Table table = getTable(DB_NAME, viewName);
Assertions.assertTrue(table instanceof View);
return (View) table;
}
}

77
fe/fe-core/src/test/java/com/starrocks/sql/optimizer/transformer/MVTransformerContextTest.java Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2021-present StarRocks, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.starrocks.sql.optimizer.transformer;
import com.starrocks.catalog.View;
import com.starrocks.sql.optimizer.rule.transformation.materialization.MVTestBase;
import com.starrocks.sql.plan.ConnectorPlanTestBase;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
public class MVTransformerContextTest extends MVTestBase {
@BeforeAll
public static void beforeClass() throws Exception {
MVTestBase.beforeClass();
starRocksAssert.withTable(cluster, "depts");
starRocksAssert.withTable(cluster, "emps");
ConnectorPlanTestBase.mockHiveCatalog(connectContext);
}
@Test
public void testIsViewBasedRewrite1() throws Exception {
connectContext.getSessionVariable().setEnableMaterializedViewTextMatchRewrite(true);
MVTransformerContext mvTransformerContext = new MVTransformerContext(connectContext, false);
Assertions.assertTrue(mvTransformerContext.isEnableTextBasedMVRewrite());
Assertions.assertFalse(mvTransformerContext.isInlineView());
Assertions.assertFalse(mvTransformerContext.isEnableViewBasedMVRewrite(null));
starRocksAssert.withView("CREATE VIEW view1 as select * from emps");
View view1 = getView("view1");
Assertions.assertFalse(mvTransformerContext.isEnableViewBasedMVRewrite(view1));
starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW mv1 " +
"DISTRIBUTED BY HASH(empid) " +
"REFRESH ASYNC " +
"AS select empid, deptno from view1");
Assertions.assertFalse(view1.getRelatedMaterializedViews().isEmpty());
Assertions.assertTrue(mvTransformerContext.isEnableViewBasedMVRewrite(view1));
starRocksAssert.dropView("view1");
starRocksAssert.dropMaterializedView("mv1");
}
@Test
public void testIsViewBasedRewrite2() throws Exception {
connectContext.getSessionVariable().setEnableMaterializedViewRewrite(false);
MVTransformerContext mvTransformerContext = new MVTransformerContext(connectContext, false);
Assertions.assertFalse(mvTransformerContext.isEnableTextBasedMVRewrite());
Assertions.assertFalse(mvTransformerContext.isInlineView());
Assertions.assertFalse(mvTransformerContext.isEnableViewBasedMVRewrite(null));
starRocksAssert.withView("CREATE VIEW view1 as select * from emps");
View view1 = getView("view1");
Assertions.assertFalse(mvTransformerContext.isEnableViewBasedMVRewrite(view1));
starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW mv1 " +
"DISTRIBUTED BY HASH(empid) " +
"REFRESH ASYNC " +
"AS select empid, deptno from view1");
Assertions.assertFalse(view1.getRelatedMaterializedViews().isEmpty());
Assertions.assertFalse(mvTransformerContext.isEnableViewBasedMVRewrite(view1));
starRocksAssert.dropView("view1");
starRocksAssert.dropMaterializedView("mv1");
}
}

25
fe/fe-core/src/test/java/com/starrocks/sql/plan/ReplayFromDumpTest.java
View File

@ -14,11 +14,13 @@
package com.starrocks.sql.plan;
import com.starrocks.catalog.View;
import com.starrocks.common.FeConstants;
import com.starrocks.common.Pair;
import com.starrocks.common.profile.Tracers;
import com.starrocks.qe.SessionVariable;
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.sql.common.QueryDebugOptions;
import com.starrocks.sql.common.StarRocksPlannerException;
import com.starrocks.sql.optimizer.OptExpression;
import com.starrocks.sql.optimizer.OptimizerContext;
@ -26,6 +28,7 @@ import com.starrocks.sql.optimizer.base.CTEProperty;
import com.starrocks.sql.optimizer.dump.QueryDumpInfo;
import com.starrocks.sql.optimizer.rule.RuleSet;
import com.starrocks.sql.optimizer.rule.transformation.JoinAssociativityRule;
import com.starrocks.sql.optimizer.transformer.MVTransformerContext;
import com.starrocks.thrift.TExplainLevel;
import com.starrocks.utframe.UtFrameUtils;
import mockit.Mock;
@ -1183,4 +1186,26 @@ public class ReplayFromDumpTest extends ReplayFromDumpTestBase {
FeConstants.USE_MOCK_DICT_MANAGER = false;
}
}
@Test
public void testViewBasedRewrite1() throws Exception {
String fileContent = getDumpInfoFromFile("query_dump/view_based_rewrite1");
QueryDumpInfo queryDumpInfo = getDumpInfoFromJson(fileContent);
SessionVariable sessionVariable = queryDumpInfo.getSessionVariable();
QueryDebugOptions debugOptions = new QueryDebugOptions();
debugOptions.setEnableQueryTraceLog(true);
sessionVariable.setQueryDebugOptions(debugOptions.toString());
new MockUp<MVTransformerContext>() {
/**
* {@link com.starrocks.sql.optimizer.transformer.MVTransformerContext#isEnableViewBasedMVRewrite(View)} ()}
*/
@Mock
public boolean isEnableViewBasedMVRewrite(View view) {
return true;
}
};
Pair<QueryDumpInfo, String> replayPair = getCostPlanFragment(fileContent, sessionVariable);
String plan = replayPair.second;
PlanTestBase.assertContains(plan, "single_mv_ads_biz_customer_combine_td_for_task_2y");
}
}

21
fe/fe-core/src/test/java/com/starrocks/sql/plan/ReplayWithMVFromDumpTest.java
View File

@ -221,12 +221,14 @@ public class ReplayWithMVFromDumpTest extends ReplayFromDumpTestBase {
@Test
public void testViewDeltaRewriter() throws Exception {
String fileContent = getDumpInfoFromFile("query_dump/view_delta");
QueryDumpInfo queryDumpInfo = getDumpInfoFromJson(fileContent);
SessionVariable sessionVariable = queryDumpInfo.getSessionVariable();
QueryDebugOptions debugOptions = new QueryDebugOptions();
debugOptions.setEnableQueryTraceLog(true);
connectContext.getSessionVariable().setQueryDebugOptions(debugOptions.toString());
sessionVariable.setQueryDebugOptions(debugOptions.toString());
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/view_delta"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
getPlanFragment(fileContent, sessionVariable, TExplainLevel.NORMAL);
Assertions.assertTrue(replayPair.second.contains("mv_yyf_trade_water3"), replayPair.second);
}
@ -299,4 +301,17 @@ public class ReplayWithMVFromDumpTest extends ReplayFromDumpTestBase {
PlanTestBase.assertContains(replayPair.second, "tbl_mock_239", "MaterializedView: true");
}
}
@Test
public void testViewBasedRewrite3() throws Exception {
String fileContent = getDumpInfoFromFile("query_dump/view_based_rewrite1");
QueryDumpInfo queryDumpInfo = getDumpInfoFromJson(fileContent);
SessionVariable sessionVariable = queryDumpInfo.getSessionVariable();
QueryDebugOptions debugOptions = new QueryDebugOptions();
debugOptions.setEnableQueryTraceLog(true);
sessionVariable.setQueryDebugOptions(debugOptions.toString());
Pair<QueryDumpInfo, String> replayPair = getPlanFragment(fileContent, sessionVariable, TExplainLevel.NORMAL);
String plan = replayPair.second;
PlanTestBase.assertContains(plan, "single_mv_ads_biz_customer_combine_td_for_task_2y");
}
}

1
fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java
View File

@ -967,6 +967,7 @@ public class UtFrameUtils {
String replaySql = initMockEnv(connectContext, replayDumpInfo);
replaySql = LogUtil.removeLineSeparator(replaySql);
Map<String, Database> dbs = null;
try {
StatementBase statementBase;
try (Timer st = Tracers.watchScope("Parse")) {

91
fe/fe-core/src/test/resources/sql/query_dump/view_based_rewrite1.json Normal file
View File

File diff suppressed because one or more lines are too long