diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index c0cd47bd5a0f2fb..9a655babb60ac05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -98,38 +98,37 @@ private static void setCatalogAndDb(ConnectContext ctx, MTMV mtmv) { public static MTMVRelation generateMTMVRelation(MTMV mtmv, ConnectContext ctx) { // Should not make table without data to empty relation when analyze the related table, // so add disable rules - SessionVariable sessionVariable = ctx.getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - if (ctx.getStatementContext() != null) { - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - Plan plan; - try { - plan = getPlanBySql(mtmv.getQuerySql(), ctx); - } finally { - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } + Plan plan = getPlanBySql(mtmv.getQuerySql(), ctx, ExplainLevel.ANALYZED_PLAN, + CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); return generateMTMVRelation(plan); } public static MTMVRelation generateMTMVRelation(Plan plan) { - return new MTMVRelation(getBaseTables(plan, true), getBaseTables(plan, false), getBaseViews(plan)); + return new MTMVRelation(getBaseTables(plan, true, true), + getBaseTables(plan, false, true), + getBaseViews(plan, true, true)); } - private static Set getBaseTables(Plan plan, boolean expand) { + private static Set getBaseTables(Plan plan, boolean expandMaterializedView, + boolean expandView) { TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( com.google.common.collect.Sets - .newHashSet(TableType.values()), expand); + .newHashSet(TableType.values()), expandMaterializedView, expandView); plan.accept(TableCollector.INSTANCE, collectorContext); Set collectedTables = collectorContext.getCollectedTables(); return transferTableIfToInfo(collectedTables); } - private static Set getBaseViews(Plan plan) { - return Sets.newHashSet(); + private static Set getBaseViews(Plan plan, boolean expandMaterializedView, + boolean expandView) { + TableCollectorContext collectorContext = + new TableCollector.TableCollectorContext( + com.google.common.collect.Sets + .newHashSet(TableType.VIEW), expandMaterializedView, expandView); + plan.accept(TableCollector.INSTANCE, collectorContext); + Set collectedTables = collectorContext.getCollectedTables(); + return transferTableIfToInfo(collectedTables); } private static Set transferTableIfToInfo(Set tables) { @@ -140,7 +139,8 @@ private static Set transferTableIfToInfo(Set tables) { return result; } - private static Plan getPlanBySql(String querySql, ConnectContext ctx) { + private static Plan getPlanBySql(String querySql, ConnectContext ctx, + ExplainLevel explainLevel, String disableRules) { List statements; try { statements = new NereidsParser().parseSQL(querySql); @@ -151,11 +151,19 @@ private static Plan getPlanBySql(String querySql, ConnectContext ctx) { LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); StatementContext original = ctx.getStatementContext(); ctx.setStatementContext(new StatementContext()); + + Set tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames(); + ctx.getSessionVariable().setDisableNereidsRules(disableRules); + if (ctx.getStatementContext() != null) { + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + } try { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.NONE); + return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, explainLevel); } finally { ctx.setStatementContext(original); + ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules)); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 2e8baecf1656390..bad966252a09af8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -80,7 +80,8 @@ public void initMaterializationContext(CascadesContext cascadesContext) { */ protected void doInitMaterializationContext(CascadesContext cascadesContext) { // Only collect the table or mv which query use directly, to avoid useless mv partition in rewrite - TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false); + TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), + false, true); try { Plan rewritePlan = cascadesContext.getRewritePlan(); // Keep use one connection context when in query, if new connect context, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index de5e188d5a65bf7..61c0fb33b2c84e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -253,8 +253,22 @@ public void analyzeQuery(ConnectContext ctx, Map mvProperties) t // this is for expression column name infer when not use alias LogicalSink logicalSink = new UnboundResultSink<>(logicalQuery); // must disable constant folding by be, because be constant folding may return wrong type - ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_FOLD_CONSTANT_BY_BE, "false"); - Plan plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + ctx.getSessionVariable().disableConstantFoldingByBEOnce(); + // Should not make table without data to empty relation when analyze the related table, + // so add disable rules + Set tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames(); + ctx.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + Plan plan; + try { + // must disable constant folding by be, because be constant folding may return wrong type + ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_FOLD_CONSTANT_BY_BE, "false"); + plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + } finally { + // after operate, roll back the disable rules + ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules)); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + } // can not contain VIEW or MTMV analyzeBaseTables(planner.getAnalyzedPlan()); // can not contain Random function @@ -265,8 +279,7 @@ public void analyzeQuery(ConnectContext ctx, Map mvProperties) t throw new AnalysisException("can not contain invalid expression"); } getRelation(planner); - this.mvPartitionInfo = mvPartitionDefinition - .analyzeAndTransferToMTMVPartitionInfo(planner, ctx, logicalQuery); + this.mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner, ctx); this.partitionDesc = generatePartitionDesc(ctx); getColumns(plan, ctx, mvPartitionInfo.getPartitionCol(), distribution); analyzeKeys(); @@ -311,24 +324,9 @@ private void analyzeKeys() { } } + // Should use analyzed plan for collect views and tables private void getRelation(NereidsPlanner planner) { - // Should not make table without data to empty relation when analyze the related table, - // so add disable rules - ConnectContext ctx = planner.getCascadesContext().getConnectContext(); - SessionVariable sessionVariable = ctx.getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - if (ctx.getStatementContext() != null) { - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - Plan plan; - try { - plan = planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.NONE); - } finally { - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - this.relation = MTMVPlanUtil.generateMTMVRelation(plan); + this.relation = MTMVPlanUtil.generateMTMVRelation(planner.getAnalyzedPlan()); } private PartitionDesc generatePartitionDesc(ConnectContext ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java index 427e2368e7ab2ba..43036cc1e31403a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java @@ -37,7 +37,6 @@ import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; import org.apache.doris.nereids.trees.expressions.Cast; @@ -45,11 +44,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import com.google.common.collect.Sets; @@ -71,11 +66,9 @@ public class MTMVPartitionDefinition { * * @param planner planner * @param ctx ctx - * @param logicalQuery logicalQuery * @return MTMVPartitionInfo */ - public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx, - LogicalPlan logicalQuery) { + public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx) { MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(partitionType); if (this.partitionType == MTMVPartitionType.SELF_MANAGE) { return mtmvPartitionInfo; @@ -99,7 +92,7 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl timeUnit = null; } mtmvPartitionInfo.setPartitionCol(partitionColName); - RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName, timeUnit); + RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, partitionColName, timeUnit); mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn()); mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo()); if (relatedTableInfo.getPartitionExpression().isPresent()) { @@ -124,47 +117,33 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl return mtmvPartitionInfo; } - private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, LogicalPlan - logicalQuery, - String partitionColName, - String timeUnit) { + // Should use rewritten plan without view and subQuery to get related partition table + private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, + String partitionColName, String timeUnit) { CascadesContext cascadesContext = planner.getCascadesContext(); - SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - // Should not make table without data to empty relation when analyze the related table, - // so add disable rules - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + + RelatedTableInfo relatedTableInfo = MaterializedViewUtils + .getRelatedTableInfo(partitionColName, timeUnit, planner.getRewrittenPlan(), cascadesContext); + if (!relatedTableInfo.isPctPossible()) { + throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning," + + " the fail reason is %s", relatedTableInfo.getFailReason())); + } + MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo()); + Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); try { - Plan mvRewrittenPlan = - planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); - RelatedTableInfo relatedTableInfo = MaterializedViewUtils - .getRelatedTableInfo(partitionColName, timeUnit, mvRewrittenPlan, cascadesContext); - if (!relatedTableInfo.isPctPossible()) { - throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning," - + " the fail reason is %s", relatedTableInfo.getFailReason())); - } - MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo()); - Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - try { - partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames()); - } catch (DdlException e) { - throw new AnalysisException(e.getMessage(), e); - } + partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames()); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage(), e); + } - if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) { - throw new AnalysisException("error related column: " + relatedTableInfo.getColumn()); - } - if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) - && partitionColumnNames.size() != 1) { - throw new AnalysisException("only hms table support multi column partition."); - } - return relatedTableInfo; - } finally { - // after operate, roll back the disable rules - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) { + throw new AnalysisException("error related column: " + relatedTableInfo.getColumn()); + } + if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) + && partitionColumnNames.size() != 1) { + throw new AnalysisException("only hms table support multi column partition."); } + return relatedTableInfo; } private static List convertToLegacyArguments(List children) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index 2e2cdb810f0f722..c3111945f968173 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -20,10 +20,12 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.catalog.View; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalView; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; import org.apache.doris.qe.ConnectContext; @@ -69,8 +71,20 @@ public Plan visitPhysicalCatalogRelation(PhysicalCatalogRelation catalogRelation return catalogRelation; } + @Override + public Plan visitLogicalView(LogicalView logicalView, TableCollectorContext context) { + View view = logicalView.getView(); + if (context.getTargetTableTypes().isEmpty() || context.getTargetTableTypes().contains(view.getType())) { + context.getCollectedTables().add(view); + } + if (context.isExpandView()) { + return super.visit(logicalView, context); + } + return logicalView; + } + private void expandMvAndCollect(MTMV mtmv, TableCollectorContext context) { - if (!context.isExpand()) { + if (!context.isExpandMaterializedView()) { return; } // Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong @@ -87,12 +101,15 @@ public static final class TableCollectorContext { private final Set collectedTables = new HashSet<>(); private final Set targetTableTypes; // if expand the mv or not - private final boolean expand; + private final boolean expandMaterializedView; + private final boolean expandView; private ConnectContext connectContext; - public TableCollectorContext(Set targetTableTypes, boolean expand) { + public TableCollectorContext(Set targetTableTypes, boolean expandMaterializedView, + boolean expandView) { this.targetTableTypes = targetTableTypes; - this.expand = expand; + this.expandMaterializedView = expandMaterializedView; + this.expandView = expandView; } public Set getCollectedTables() { @@ -103,8 +120,8 @@ public Set getTargetTableTypes() { return targetTableTypes; } - public boolean isExpand() { - return expand; + public boolean isExpandMaterializedView() { + return expandMaterializedView; } public ConnectContext getConnectContext() { @@ -114,5 +131,9 @@ public ConnectContext getConnectContext() { public void setConnectContext(ConnectContext connectContext) { this.connectContext = connectContext; } + + public boolean isExpandView() { + return expandView; + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java index 60f6e19faab6350..c87aa254273b823 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java @@ -102,10 +102,28 @@ protected void runBeforeAll() throws Exception { + "\"replication_num\" = \"1\"\n" + ");"); + createTable("CREATE TABLE `table5` (\n" + + " `c1` bigint(20) NULL,\n" + + " `c2` bigint(20) NULL,\n" + + " `c3` bigint(20) not NULL,\n" + + " `c4` DATE not NULL,\n" + + " `k4` bitmap BITMAP_UNION ,\n" + + " `k5` bitmap BITMAP_UNION \n" + + ") ENGINE=OLAP\n" + + "AGGREGATE KEY(`c1`, `c2`, `c3`, `c4`)\n" + + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + createView("CREATE VIEW `view1` AS SELECT t1.*, random() FROM\n" + "`table1` t1 LEFT JOIN\n" + "`table2` t2 ON t1.c1 = t2.c1;"); + createView("CREATE VIEW `view2` AS SELECT table5.c1, view1.c2 FROM view1\n" + + "LEFT JOIN table5 ON table5.c1 = view1.c1;\n"); + createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + "DISTRIBUTED BY RANDOM BUCKETS 1\n" + "PROPERTIES ('replication_num' = '1') \n" @@ -130,7 +148,7 @@ public void test1() { Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Random); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, true); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -159,7 +177,7 @@ public void test2() { Assertions.assertTrue(nondeterministicFunctionSet.get(1) instanceof Random); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, true); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -172,6 +190,204 @@ public void test2() { }); } + @Test + public void testCollectTable() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); + new MockUp() { + @Mock + public BitSet getDisableNereidsRules() { + return disableNereidsRules; + } + }; + PlanChecker.from(connectContext) + .checkExplain("SELECT view2.* FROM " + + "view2 " + + "LEFT JOIN mv1 ON mv1.c1 = view2.c1 " + + "WHERE view2.c1 IN (SELECT c1 FROM table2)", + nereidsPlanner -> { + Plan analyzedPlan = nereidsPlanner.getAnalyzedPlan(); + // Check nondeterministic collect + List nondeterministicFunctionSet = + MaterializedViewUtils.extractNondeterministicFunction(analyzedPlan); + Assertions.assertEquals(1, nondeterministicFunctionSet.size()); + Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Random); + // Expand view and materialized view, only collect olap + // view 1 contains table1 and table2, view2 contains view1 and table5 + // mv1 contains table1 and table3 + TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.OLAP), true, true); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + Set expectedTables = new HashSet<>(); + expectedTables.add("table1"); + expectedTables.add("table2"); + expectedTables.add("table3"); + expectedTables.add("table5"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.VIEW), true, true); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("view1"); + expectedTables.add("view2"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + // Expand view and materialized view, only all type table + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(), true, true); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("table1"); + expectedTables.add("table2"); + expectedTables.add("table3"); + expectedTables.add("table5"); + expectedTables.add("view1"); + expectedTables.add("view2"); + expectedTables.add("mv1"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + // Expand view but not materialized view, only collect olap + // view 1 contains table1 and table2, view2 contains view1 and table5 + // mv1 contains table1 and table3 + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.OLAP), false, true); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("table1"); + expectedTables.add("table2"); + expectedTables.add("table5"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.VIEW), false, true); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("view1"); + expectedTables.add("view2"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + // Expand view but not materialized view, collect all type table + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(), false, true); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("table1"); + expectedTables.add("table2"); + expectedTables.add("table5"); + expectedTables.add("mv1"); + expectedTables.add("view1"); + expectedTables.add("view2"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + // Expand materialized view but not view, only collect olap + // view 1 contains table1 and table2, view2 contains view1 and table5 + // mv1 contains table1 and table3 + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.OLAP), true, false); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("table1"); + expectedTables.add("table2"); + expectedTables.add("table3"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.VIEW), true, false); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("view2"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + // Expand materialized view but not view, collect all type table + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(), true, false); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("table1"); + expectedTables.add("table2"); + expectedTables.add("table3"); + expectedTables.add("view2"); + expectedTables.add("mv1"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + // Not expand materialized view and view, only collect olap + // view 1 contains table1 and table2, view2 contains view1 and table5 + // mv1 contains table1 and table3 + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.OLAP), false, false); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("table2"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.VIEW), false, false); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("view2"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + + // Not expand materialized view and view, collect all type table + collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(), false, false); + analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); + expectedTables = new HashSet<>(); + expectedTables.add("table2"); + expectedTables.add("view2"); + expectedTables.add("mv1"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + }); + } + @Test public void test3() throws Exception { connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); @@ -196,7 +412,7 @@ public BitSet getDisableNereidsRules() { Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Uuid); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, true); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -210,7 +426,7 @@ public BitSet getDisableNereidsRules() { TableCollectorContext collectorContextWithNoExpand = new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP), - false); + false, true); physicalPlan.accept(TableCollector.INSTANCE, collectorContextWithNoExpand); Set expectedTablesWithNoExpand = new HashSet<>(); expectedTablesWithNoExpand.add("table1"); @@ -222,7 +438,7 @@ public BitSet getDisableNereidsRules() { expectedTablesWithNoExpand); TableCollectorContext mvCollectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.MATERIALIZED_VIEW), true); + Sets.newHashSet(TableType.MATERIALIZED_VIEW), true, true); physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContext); Set expectedMvs = new HashSet<>(); expectedMvs.add("mv1"); @@ -234,7 +450,8 @@ public BitSet getDisableNereidsRules() { TableCollectorContext mvCollectorContextWithNoExpand = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.MATERIALIZED_VIEW), false); + Sets.newHashSet(TableType.MATERIALIZED_VIEW), false, + true); physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContextWithNoExpand); Set expectedMvsWithNoExpand = new HashSet<>(); expectedMvsWithNoExpand.add("mv1"); @@ -246,7 +463,8 @@ public BitSet getDisableNereidsRules() { TableCollectorContext allTableTypeWithExpand = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.values()), true); + Sets.newHashSet(TableType.values()), true, + true); physicalPlan.accept(TableCollector.INSTANCE, allTableTypeWithExpand); // when collect in plan with expand, should collect table which is expended Set expectedTablesWithExpand = new HashSet<>(); diff --git a/regression-test/data/nereids_rules_p0/mv/with_cte/with_cte.out b/regression-test/data/nereids_rules_p0/mv/with_cte/with_cte.out new file mode 100644 index 000000000000000..55b1a3e5091e2da --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/with_cte/with_cte.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +4 + +-- !query1_0_after -- +4 + diff --git a/regression-test/suites/nereids_rules_p0/mv/with_cte/with_cte.groovy b/regression-test/suites/nereids_rules_p0/mv/with_cte/with_cte.groovy new file mode 100644 index 000000000000000..9e12b69fdbe4e75 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/with_cte/with_cte.groovy @@ -0,0 +1,199 @@ +package mv.with_cte +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("rewrite_with_cte") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) ( + PARTITION `day_1` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_2` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_3` VALUES LESS THAN ("2023-12-30")) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp with sync""" + sql """analyze table lineitem with sync""" + sql """analyze table orders with sync""" + + + def mv1_0 = + """ + select + distinct + o_orderkey, + o_orderdate + from orders + where O_COMMENT not in ('mi', 'mm'); + """ + def query1_0 = + """ + select + count(*) + from + ( + with view1 as ( + select + distinct o_orderkey, + o_orderdate + from + orders + where + O_COMMENT not in ('mi', 'mm') + and 'BI' = 'BI' + ), + view2 as ( + select + distinct o_orderkey, + o_orderdate + from + view1 + where + o_orderdate = '2023-12-09' + ) + select + * + from + view1 + union all + select + * + from + view2 + ) as t + limit + 3; + """ + order_qt_query1_0_before "${query1_0}" + // Test mv rewrite when cte + async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" +// sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" +}