tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[BugFix] Fix view based mv rewrite bug (backport #62198) (#62228) 

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
Co-authored-by: shuming.li <ming.moriarty@gmail.com>
This commit is contained in:
mergify[bot] 2025-08-22 13:13:10 +00:00 committed by GitHub
parent 484afe81e1
commit 34e3f8ca7b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 710 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/en/sql-reference/System_variable.md
View File

@ -306,6 +306,12 @@ Used for MySQL client compatibility. No practical usage.
* **Default**: true
* **Introduced in**: v2.5.13, v3.0.7, v3.1.4, v3.2.0, v3.3.0
### enable_cbo_based_mv_rewrite
* **Description**: Whether to enable materialized view rewrite in CBO phase which can maximize the likelihood of successful query rewriting (e.g., when the join order differs between materialized views and queries), but it will increase the execution time of the optimizer phase.
* **Default**: true
* **Introduced in**: v3.5.5, v4.0.1
### enable_parquet_reader_bloom_filter
* **Description**: Whether to enable the bloom filter of Parquet file to improve performance. `true` indicates enabling the bloom filter, and `false` indicates disabling it. You can also control this behavior on system level using the BE configuration `parquet_reader_bloom_filter_enable`. Bloom filters in Parquet are maintained **at the column level within each row group**. If a Parquet file contains bloom filters for certain columns, queries can use predicates on those columns to efficiently skip row groups.

5
docs/ja/sql-reference/System_variable.md
View File

@ -298,6 +298,11 @@ MySQL クライアント互換性のために使用されます。実際の用
* **デフォルト**: true
* **導入バージョン**: v2.5.13, v3.0.7, v3.1.4, v3.2.0, v3.3.0
### enable_cbo_based_mv_rewrite
* **説明**: CBO フェーズでマテリアライズドビューの書き換えを有効にするかどうか。これにより、クエリ書き換えの成功率を最大化できます（例：マテリアライズドビューとクエリの結合順序が異なる場合）が、オプティマイザフェーズの実行時間が増加します。
* **デフォルト**: true
* **導入バージョン**: v3.5.5, v4.0.1
### enable_parquet_reader_bloom_filter
* **説明**: パフォーマンスを向上させるために Parquet ファイルのブルームフィルターを有効にするかどうかを制御するブール値。`true` はブルームフィルタを有効にすることを示し、`false` は無効にすることを示す。BE 設定 `parquet_reader_bloom_filter_enable` を使用して、システムレベルでこの動作を制御することもできます。Parquet におけるブルームフィルタは、**各行グループ内のカラムレベルで管理されます**。Parquet ファイルに特定の列に対するブルームフィルタが含まれている場合、クエリはそれらの列に対する述語を使用して行グループを効率的にスキップすることができます。

6
docs/zh/sql-reference/System_variable.md
View File

@ -303,6 +303,12 @@ ALTER USER 'jack' SET PROPERTIES ('session.query_timeout' = '600');
* 默认值：true
* 引入版本：v2.5.13，v3.0.7，v3.1.4，v3.2.0，v3.3.0
### enable_cbo_based_mv_rewrite
* 描述：是否在 CBO 阶段启用物化视图改写，这可以最大化查询改写成功的可能性（例如，当物化视图和查询之间的连接顺序不同时），但这会增加优化器阶段的执行时间。
* 默认值：true
* 引入版本：v3.5.5，v4.0.1
### enable_parquet_reader_bloom_filter
* 描述：是否启用 Parquet 文件的 Bloom Filter 以提高性能。`true` 表示启用 Bloom Filter，`false` 表示禁用。还可以使用 BE 参数 `parquet_reader_bloom_filter_enable` 在 Session 级别上控制这一行为。Parquet 中的 Bloom Filter 是在**每个行组的列级维护的**。如果 Parquet 文件包含某些列的 Bloom Filter，查询就可以使用这些列上的谓词来有效地跳过行组。

14
fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
View File

@ -704,6 +704,7 @@ public class SessionVariable implements Serializable, Writable, Cloneable {
public static final String ENABLE_VIEW_BASED_MV_REWRITE = "enable_view_based_mv_rewrite";
public static final String ENABLE_CBO_VIEW_BASED_MV_REWRITE = "enable_cbo_view_based_mv_rewrite";
public static final String ENABLE_CBO_BASED_MV_REWRITE = "enable_cbo_based_mv_rewrite";
public static final String ENABLE_SPM_REWRITE = "enable_spm_rewrite";
public static final String SPM_REWRITE_TIMEOUT_MS = "spm_rewrite_timeout_ms";
@ -2313,6 +2314,11 @@ public class SessionVariable implements Serializable, Writable, Cloneable {
@VarAttr(name = ENABLE_CBO_VIEW_BASED_MV_REWRITE)
private boolean enableCBOViewBasedMvRewrite = false;
// Whether enable mv rewrite in CBO phase, true by default which means will try best to use mv
// to rewrite in RBO and CBO phase.
@VarAttr(name = ENABLE_CBO_BASED_MV_REWRITE)
private boolean enableCBOBasedMVRewrite = true;
@VarAttr(name = ENABLE_SPM_REWRITE)
private boolean enableSPMRewrite = false;
@ -4458,6 +4464,14 @@ public class SessionVariable implements Serializable, Writable, Cloneable {
return this.enableCBOViewBasedMvRewrite;
}
public boolean isEnableCBOBasedMVRewrite() {
return enableCBOBasedMVRewrite;
}
public void setEnableCboBasedMvRewrite(boolean enableCBOBasedMVRewrite) {
this.enableCBOBasedMVRewrite = enableCBOBasedMVRewrite;
}
public int getCboMaterializedViewRewriteRuleOutputLimit() {
return cboMaterializedViewRewriteRuleOutputLimit;
}

16
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java
View File

@ -248,10 +248,6 @@ public class QueryOptimizer extends Optimizer {
Preconditions.checkNotNull(memo);
memo.init(logicOperatorTree);
if (mvRewriteStrategy.enableViewBasedRewrite && context.getQueryMaterializationContext() != null) {
// LogicalTreeWithView is logically equivalent to logicOperatorTree
addViewBasedPlanIntoMemo(context.getQueryMaterializationContext().getQueryOptPlanWithView());
}
OptimizerTraceUtil.log("after logical rewrite, root group:\n%s", memo.getRootGroup());
// Currently, we cache output columns in logic property.
@ -322,14 +318,6 @@ public class QueryOptimizer extends Optimizer {
}
}
private void addViewBasedPlanIntoMemo(OptExpression logicalTreeWithView) {
if (logicalTreeWithView == null) {
return;
}
Memo memo = context.getMemo();
memo.copyIn(memo.getRootGroup(), logicalTreeWithView);
}
private void prepareMvRewrite(ConnectContext connectContext, OptExpression logicOperatorTree,
ColumnRefFactory columnRefFactory, ColumnRefSet requiredColumns) {
SessionVariable sessionVariable = connectContext.getSessionVariable();
@ -765,6 +753,7 @@ public class QueryOptimizer extends Optimizer {
try (Timer ignored = Tracers.watchScope("MVViewRewrite")) {
OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "try VIEW_BASED_MV_REWRITE");
OptExpression treeWithView = queryMaterializationContext.getQueryOptPlanWithView();
OptimizerTraceUtil.logOptExpression("before ViewBasedMvRuleRewrite:\n%s", tree);
// should add a LogicalTreeAnchorOperator for rewrite
treeWithView = OptExpression.create(new LogicalTreeAnchorOperator(), treeWithView);
if (mvRewriteStrategy.enableMultiTableRewrite) {
@ -791,6 +780,7 @@ public class QueryOptimizer extends Optimizer {
}
OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "original view scans size: {}, " +
"left view scans size: {}", origQueryViewScanOperators.size(), leftViewScanOperators.size());
OptimizerTraceUtil.logOptExpression("after ViewBasedMvRuleRewrite:\n%s", tree);
} catch (Exception e) {
OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE",
"single table view based mv rule rewrite failed.", e);
@ -954,7 +944,7 @@ public class QueryOptimizer extends Optimizer {
context.getRuleSet().addRealtimeMVRules();
}
if (mvRewriteStrategy.enableMultiTableRewrite) {
if (mvRewriteStrategy.enableCBOBasedMvRewrite && mvRewriteStrategy.enableMultiTableRewrite) {
context.getRuleSet().addSingleTableMvRewriteRule();
context.getRuleSet().addMultiTableMvRewriteRule();
}

15
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/logical/LogicalViewScanOperator.java
View File

@ -15,6 +15,7 @@
package com.starrocks.sql.optimizer.operator.logical;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import com.starrocks.analysis.Expr;
import com.starrocks.catalog.Column;
import com.starrocks.catalog.Table;
@ -26,6 +27,7 @@ import com.starrocks.sql.optimizer.operator.Operator;
import com.starrocks.sql.optimizer.operator.OperatorType;
import com.starrocks.sql.optimizer.operator.OperatorVisitor;
import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils;
import java.util.Map;
@ -42,6 +44,9 @@ public class LogicalViewScanOperator extends LogicalScanOperator {
// Original plan evaluator(inlined view) for the input logical plan tree which has not done rule based rewrite yet,
// this is only used when the view scan operator cannot be rewritten by view-based rewrite rules.
private OptPlanEvaluator optPlanEvaluator;
// this maps original inlined view's column ref to non-inlined column ref which can be used for
// replacing from non-inlined column refs to inlined view's column ref.
private Map<ColumnRefOperator, ScalarOperator> originalColumnRefToInlinedColumnRefMap;
public LogicalViewScanOperator(
int relationId,
@ -49,12 +54,15 @@ public class LogicalViewScanOperator extends LogicalScanOperator {
Map<ColumnRefOperator, Column> colRefToColumnMetaMap,
Map<Column, ColumnRefOperator> columnMetaToColRefMap,
ColumnRefSet outputColumnSet,
Map<Expr, ColumnRefOperator> expressionToColumns) {
Map<Expr, ColumnRefOperator> expressionToColumns,
Map<ColumnRefOperator, ScalarOperator> originalColumnRefToInlinedColumnRefMap) {
super(OperatorType.LOGICAL_VIEW_SCAN, table, colRefToColumnMetaMap,
columnMetaToColRefMap, Operator.DEFAULT_LIMIT, null, null);
this.relationId = relationId;
this.outputColumnSet = outputColumnSet;
this.expressionToColumns = expressionToColumns;
// copy the originalColumnRefToInlinedColumnRefMap to avoid modification by projection
this.originalColumnRefToInlinedColumnRefMap = Maps.newHashMap(originalColumnRefToInlinedColumnRefMap);
}
private LogicalViewScanOperator() {
@ -84,6 +92,10 @@ public class LogicalViewScanOperator extends LogicalScanOperator {
return this.optPlanEvaluator.evaluate();
}
public Map<ColumnRefOperator, ScalarOperator> getOriginalColumnRefToInlinedColumnRefMap() {
return originalColumnRefToInlinedColumnRefMap;
}
/**
* Evaluate the original plan and return the optimized plan for the input logical plan tree which is with inlined view and
* has not done rule based rewrite yet.
@ -157,6 +169,7 @@ public class LogicalViewScanOperator extends LogicalScanOperator {
builder.expressionToColumns = scanOperator.expressionToColumns;
builder.outputColumnSet = scanOperator.outputColumnSet;
builder.optPlanEvaluator = scanOperator.optPlanEvaluator;
builder.originalColumnRefToInlinedColumnRefMap = scanOperator.originalColumnRefToInlinedColumnRefMap;
return this;
}
}

6
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java
View File

@ -72,6 +72,7 @@ public class MvRewriteStrategy {
public boolean enableViewBasedRewrite = false;
public boolean enableSingleTableRewrite = false;
public boolean enableMultiTableRewrite = false;
public boolean enableCBOBasedMvRewrite = false;
static class MvStrategyArbitrator {
private final OptimizerOptions optimizerOptions;
@ -134,7 +135,7 @@ public class MvRewriteStrategy {
return true;
}
private boolean isEnableCBOMultiTableRewrite(OptExpression queryPlan) {
private boolean isEnableMultiTableRewrite(OptExpression queryPlan) {
if (!sessionVariable.isEnableMaterializedViewSingleTableViewDeltaRewrite() &&
MvUtils.getAllTables(queryPlan).size() <= 1) {
return false;
@ -174,7 +175,8 @@ public class MvRewriteStrategy {
strategy.enableSingleTableRewrite = arbitrator.isEnableRBOSingleTableRewrite(queryPlan);
// cbo strategies
strategy.enableMultiTableRewrite = arbitrator.isEnableCBOMultiTableRewrite(queryPlan);
strategy.enableCBOBasedMvRewrite = sessionVariable.isEnableCBOBasedMVRewrite();
strategy.enableMultiTableRewrite = arbitrator.isEnableMultiTableRewrite(queryPlan);
return strategy;
}

64
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java
View File

@ -73,6 +73,7 @@ import com.starrocks.sql.optimizer.Optimizer;
import com.starrocks.sql.optimizer.OptimizerContext;
import com.starrocks.sql.optimizer.OptimizerFactory;
import com.starrocks.sql.optimizer.OptimizerOptions;
import com.starrocks.sql.optimizer.OptimizerTraceUtil;
import com.starrocks.sql.optimizer.QueryMaterializationContext;
import com.starrocks.sql.optimizer.Utils;
import com.starrocks.sql.optimizer.base.ColumnRefFactory;
@ -1193,11 +1194,17 @@ public class MvUtils {
// add a LogicalTreeAnchorOperator to replace the tree easier
OptExpression anchorExpr = OptExpression.create(new LogicalTreeAnchorOperator(), queryExpression);
doReplaceLogicalViewScanOperator(anchorExpr, 0, queryExpression);
// check if there is any view scan operator in the tree
List<Operator> viewScanOperators = Lists.newArrayList();
MvUtils.collectViewScanOperator(anchorExpr, viewScanOperators);
if (!viewScanOperators.isEmpty()) {
OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE",
"After replacing logical view scan operator but found view scan operator in the tree, "
+ "so cannot rewrite the query expression: " + queryExpression);
return null;
}
OptExpression newQuery = anchorExpr.inputAt(0);
deriveLogicalProperty(newQuery);
return newQuery;
@ -1209,29 +1216,48 @@ public class MvUtils {
LogicalOperator op = queryExpression.getOp().cast();
if (op instanceof LogicalViewScanOperator) {
LogicalViewScanOperator viewScanOperator = op.cast();
OptExpression viewPlan = viewScanOperator.getOriginalPlanEvaluator();
OptExpression inlineViewPlan = viewScanOperator.getOriginalPlanEvaluator();
if (viewScanOperator.getPredicate() != null) {
// If viewScanOperator contains predicate, we need to rewrite them,
// otherwise predicate will be lost.
Map<ColumnRefOperator, ScalarOperator> reverseColumnRefMap =
viewScanOperator.getProjection().getColumnRefMap().entrySet().stream()
.collect(Collectors.toMap(e -> (ColumnRefOperator) e.getValue(), e -> e.getKey()));
ReplaceColumnRefRewriter rewriter = new ReplaceColumnRefRewriter(reverseColumnRefMap);
Operator.Builder builder = OperatorBuilderFactory.build(viewPlan.getOp());
builder.withOperator(viewPlan.getOp());
// rewrite predicate
builder.setPredicate(rewriter.rewrite(viewScanOperator.getPredicate()));
// rewrite projection
Map<ColumnRefOperator, ScalarOperator> newColumnRefMap = viewScanOperator.getProjection().getColumnRefMap()
// original map records inlined view's column ref to non-inlined view's column ref mapping,
// now we need to rewrite non-inlined view's column ref to inlined view's column ref,
// so we need to reverse the mapping.
Map<ColumnRefOperator, ScalarOperator> originalColumnRefToInlinedColumnRefMap =
Maps.newHashMap(viewScanOperator.getOriginalColumnRefToInlinedColumnRefMap());
// add new added column ref mapping
if (viewScanOperator.getProjection() != null) {
viewScanOperator.getProjection().getColumnRefMap()
.entrySet()
.stream()
.forEach(e -> originalColumnRefToInlinedColumnRefMap.put(e.getKey(), e.getValue()));
}
Map<ColumnRefOperator, ScalarOperator> reverseColumnRefMap = originalColumnRefToInlinedColumnRefMap
.entrySet()
.stream()
.map(e -> Maps.immutableEntry(e.getKey(), rewriter.rewrite(e.getValue())))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
builder.setProjection(new Projection(newColumnRefMap));
Operator newViewPlanOp = builder.build();
parent.setChild(index, OptExpression.create(newViewPlanOp, viewPlan.getInputs()));
.collect(Collectors.toMap(e -> (ColumnRefOperator) e.getValue(), e -> e.getKey()));
// If viewScanOperator contains predicate, we need to rewrite them, otherwise predicate will be lost.
ReplaceColumnRefRewriter rewriter = new ReplaceColumnRefRewriter(reverseColumnRefMap);
Operator.Builder builder = OperatorBuilderFactory.build(inlineViewPlan.getOp());
builder.withOperator(inlineViewPlan.getOp());
if (viewScanOperator.getPredicate() != null) {
// rewrite predicate
ScalarOperator rewrittenPredicate = rewriter.rewrite(viewScanOperator.getPredicate());
builder.setPredicate(rewrittenPredicate);
}
// rewrite projection
if (viewScanOperator.getPredicate() != null) {
Map<ColumnRefOperator, ScalarOperator> newColumnRefMap = viewScanOperator
.getProjection().getColumnRefMap()
.entrySet()
.stream()
.map(e -> Maps.immutableEntry(e.getKey(), rewriter.rewrite(e.getValue())))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
builder.setProjection(new Projection(newColumnRefMap));
}
Operator newInlineViewPlanOp = builder.build();
parent.setChild(index, OptExpression.create(newInlineViewPlanOp, inlineViewPlan.getInputs()));
} else {
parent.setChild(index, viewPlan);
parent.setChild(index, inlineViewPlan);
}
return;
}

2
fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/RelationTransformer.java
View File

@ -946,7 +946,7 @@ public class RelationTransformer implements AstVisitor<LogicalPlan, ExpressionMa
LogicalViewScanOperator scanOperator = new LogicalViewScanOperator(relationId,
node.getView(), columnRefOperatorToColumn, columnMetaToColRefMap,
new ColumnRefSet(logicalPlan.getOutputColumn()), newExprMapping);
new ColumnRefSet(logicalPlan.getOutputColumn()), newExprMapping, projectionMap);
if (inlineView) {
// add a projection to make sure output columns keep the same,
// because LogicalViewScanOperator should be logically equivalent to logicalPlan

51
fe/fe-core/src/test/java/com/starrocks/sql/plan/ReplayWithMVFromDumpTest.java
View File

@ -248,4 +248,55 @@ public class ReplayWithMVFromDumpTest extends ReplayFromDumpTestBase {
" 3:Project\n" +
" | <slot 59> : coalesce(80: count, 0)");
}
@Test
public void testViewBasedRewrite1() throws Exception {
QueryDebugOptions debugOptions = new QueryDebugOptions();
debugOptions.setEnableQueryTraceLog(true);
SessionVariable sessionVariable = connectContext.getSessionVariable();
sessionVariable.setQueryDebugOptions(debugOptions.toString());
// disable cbo based mv rewrite
{
sessionVariable.setEnableCboBasedMvRewrite(false);
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/materialized-view/view_based_rewrite1"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
PlanTestBase.assertContains(replayPair.second, "tbl_mock_255", "MaterializedView: true");
}
// enable cbo based mv rewrite
{
sessionVariable.setEnableCboBasedMvRewrite(true);
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/materialized-view/view_based_rewrite1"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
PlanTestBase.assertContains(replayPair.second, "tbl_mock_255", "MaterializedView: true");
}
}
@Test
public void testViewBasedRewrite2() throws Exception {
QueryDebugOptions debugOptions = new QueryDebugOptions();
debugOptions.setEnableQueryTraceLog(true);
SessionVariable sessionVariable = connectContext.getSessionVariable();
sessionVariable.setQueryDebugOptions(debugOptions.toString());
// disable cbo based mv rewrite
sessionVariable.setEnableViewBasedMvRewrite(true);
{
sessionVariable.setEnableCboBasedMvRewrite(false);
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/materialized-view/view_based_rewrite2"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
PlanTestBase.assertContains(replayPair.second, "tbl_mock_239", "MaterializedView: true");
}
// enable cbo based mv rewrite
{
sessionVariable.setEnableCboBasedMvRewrite(true);
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/materialized-view/view_based_rewrite2"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
PlanTestBase.assertContains(replayPair.second, "tbl_mock_239", "MaterializedView: true");
}
}
}

279
fe/fe-core/src/test/resources/sql/query_dump/materialized-view/view_based_rewrite1.json Normal file
View File

File diff suppressed because one or more lines are too long

282
fe/fe-core/src/test/resources/sql/query_dump/materialized-view/view_based_rewrite2.json Normal file
View File

File diff suppressed because one or more lines are too long