From 1662e4709a7c2a216f840a6713612ecc0238d359 Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 9 Dec 2024 11:46:48 +0800 Subject: [PATCH] Pick some pr to 21 #43010 #43030 #43785 #44779 #44786 #44857 (#45129) --- .../java/org/apache/doris/mtmv/MTMVCache.java | 14 +- .../org/apache/doris/mtmv/MTMVPlanUtil.java | 37 +- .../joinorder/hypergraph/HyperElement.java | 27 + .../jobs/joinorder/hypergraph/edge/Edge.java | 4 +- .../hypergraph/node/AbstractNode.java | 8 +- .../nereids/rules/analysis/BindRelation.java | 4 + .../exploration/mv/HyperGraphComparator.java | 103 +++- .../mv/InitMaterializationContextHook.java | 13 +- .../mv/LogicalCompatibilityContext.java | 78 ++- .../exploration/mv/MaterializedViewUtils.java | 69 ++- .../rules/exploration/mv/StructInfo.java | 143 ++++-- .../rules/rewrite/PruneEmptyPartition.java | 7 +- .../rules/rewrite/PruneOlapScanTablet.java | 6 +- .../commands/UpdateMvByPartitionCommand.java | 9 +- .../plans/commands/info/CreateMTMVInfo.java | 44 +- .../info/MTMVPartitionDefinition.java | 71 +-- .../trees/plans/logical/LogicalOlapScan.java | 55 +- .../trees/plans/visitor/TableCollector.java | 34 +- .../rewrite/PruneOlapScanTabletTest.java | 21 +- .../doris/nereids/sqltest/SqlTestBase.java | 3 + .../nereids/trees/plans/PlanVisitorTest.java | 14 +- .../data/mv_p0/ssb/q_1_1/q_1_1.out | 11 +- .../aggregate_without_roll_up.out | 24 + .../mv/micro_test/micro_test_when_cte.out | 128 +++++ .../doris/regression/suite/Suite.groovy | 19 +- .../agg_have_dup_base.groovy | 2 + .../agg_state/test_agg_state_max_by.groovy | 3 + .../mv_p0/case_ignore/case_ignore.groovy | 3 + .../suites/mv_p0/count_star/count_star.groovy | 2 + .../suites/mv_p0/dis_26495/dis_26495.groovy | 2 + .../suites/mv_p0/k1ap2spa/k1ap2spa.groovy | 2 + .../suites/mv_p0/k1s2m3/k1s2m3.groovy | 1 + .../k1s2m3_auto_inc/k1s2m3_auto_inc.groovy | 1 + .../multi_agg_with_same_slot.groovy | 1 + .../multi_slot_k123p/multi_slot_k123p.groovy | 1 + .../multi_slot_k1a2p2ap3p.groovy | 1 + .../multi_slot_k1a2p2ap3ps.groovy | 1 + .../multi_slot_k1p2ap3p.groovy | 1 + .../multi_slot_k1p2ap3ps.groovy | 1 + .../mv_p0/mv_with_view/mv_with_view.groovy | 26 +- .../mv_p0/null_insert/null_insert.groovy | 1 + .../routine_load_hll/routine_load_hll.groovy | 1 + .../multiple_ssb_between.groovy | 51 +- .../suites/mv_p0/ssb/q_1_1/q_1_1.groovy | 40 +- .../suites/mv_p0/ssb/q_2_1/q_2_1.groovy | 2 + .../suites/mv_p0/ssb/q_3_1/q_3_1.groovy | 27 +- .../suites/mv_p0/ssb/q_4_1/q_4_1.groovy | 2 + .../suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy | 1 + .../suites/mv_p0/sum_count/sum_count.groovy | 1 + .../sum_divede_count/sum_devide_count.groovy | 2 + .../suites/mv_p0/test_28741/test_28741.groovy | 2 +- .../test_approx_count_distinct.groovy | 1 + .../suites/mv_p0/test_base/test_base.groovy | 1 + .../mv_p0/test_casewhen/test_casewhen.groovy | 1 + .../test_create_mv/test_create_mv.groovy | 2 + .../test_create_mv_complex_type.groovy | 2 + .../mv_p0/test_doc_e4/test_doc_e4.groovy | 1 + .../test_dup_group_by_mv_abs.groovy | 1 + .../test_dup_group_by_mv_plus.groovy | 1 + .../test_dup_mv_abs/test_dup_mv_abs.groovy | 1 + .../test_dup_mv_bin/test_dup_mv_bin.groovy | 1 + .../test_dup_mv_bitmap_hash.groovy | 3 + .../test_dup_mv_plus/test_dup_mv_plus.groovy | 2 +- .../test_dup_mv_repeat.groovy | 1 + .../test_dup_mv_year/test_dup_mv_year.groovy | 1 + .../suites/mv_p0/test_mv_dp/test_mv_dp.groovy | 2 + .../mv_p0/test_mv_mor/test_mv_mor.groovy | 2 + .../suites/mv_p0/test_ndv/test_ndv.groovy | 1 + .../suites/mv_p0/test_nvl/test_nvl.groovy | 1 + .../suites/mv_p0/test_o2/test_o2.groovy | 1 + .../mv_p0/test_substr/test_substr.groovy | 1 + .../mv_p0/test_tbl_name/test_tbl_name.groovy | 1 + .../test_upper_alias/test_upper_alias.groovy | 1 + .../test_user_activity.groovy | 1 + .../suites/mv_p0/unique/unique.groovy | 1 + .../testAggQueryOnAggMV1.groovy | 1 + .../testAggQueryOnAggMV10.groovy | 1 + .../testAggQueryOnAggMV11.groovy | 1 + .../testAggQueryOnAggMV2.groovy | 1 + .../testAggQueryOnAggMV3.groovy | 2 +- .../testAggQuqeryOnAggMV5.groovy | 1 + .../testAggQuqeryOnAggMV6.groovy | 1 + .../testAggQuqeryOnAggMV7.groovy | 2 + ...testAggregateMVCalcAggFunctionQuery.groovy | 1 + .../testBitmapUnionInQuery.groovy | 2 +- .../testCountDistinctToBitmap.groovy | 4 + .../testIncorrectMVRewriteInSubquery.groovy | 1 + .../testIncorrectRewriteCountDistinct.groovy | 1 + .../testJoinOnLeftProjectToJoin.groovy | 3 + .../mv_p0/ut/testNDVToHll/testNDVToHll.groovy | 1 + .../testOrderByQueryOnProjectView.groovy | 1 + .../testProjectionMV1.groovy | 1 + .../testProjectionMV2.groovy | 1 + .../testProjectionMV3.groovy | 1 + .../testProjectionMV4.groovy | 1 + .../ut/testQueryOnStar/testQueryOnStar.groovy | 1 + .../testSelectMVWithTableAlias.groovy | 2 + .../testSingleMVMultiUsage.groovy | 1 + .../mv_p0/ut/testSubQuery/testSubQuery.groovy | 2 +- .../testUnionDistinct.groovy | 2 + .../mv/agg_on_none_agg/agg_on_none_agg.groovy | 6 + .../mv/agg_variety/agg_variety.groovy | 6 + .../aggregate_with_roll_up.groovy | 3 + .../aggregate_without_roll_up.groovy | 469 +++++++++++++++++- .../mv/availability/grace_period.groovy | 2 + .../materialized_view_switch.groovy | 22 +- .../mv/dimension/dimension_1.groovy | 3 + .../mv/dimension/dimension_2_3.groovy | 3 + .../mv/dimension/dimension_2_4.groovy | 4 + .../mv/dimension/dimension_2_5.groovy | 4 + .../mv/dimension/dimension_2_6.groovy | 6 + .../mv/dimension/dimension_2_full_join.groovy | 2 + .../dimension/dimension_2_inner_join.groovy | 3 + .../dimension_2_left_anti_join.groovy | 3 + .../mv/dimension/dimension_2_left_join.groovy | 3 + .../dimension_2_left_semi_join.groovy | 3 + .../dimension_2_right_anti_join.groovy | 3 + .../dimension/dimension_2_right_join.groovy | 3 + .../dimension_2_right_semi_join.groovy | 3 + .../mv/dimension/dimension_self_conn.groovy | 3 + .../dimension_2_join_agg.groovy | 15 +- .../dimension_join_agg_negative.groovy | 3 + .../filter_equal_or_notequal.groovy | 3 + .../mv/direct_query/direct_query.groovy | 3 + .../mv/grouping_sets/grouping_sets.groovy | 3 + .../is_in_debug_mode/is_in_debug_mode.groovy | 156 ++++++ .../join/dphyp_inner/inner_join_dphyp.groovy | 4 + .../join/dphyp_outer/outer_join_dphyp.groovy | 4 + .../mv/join/inner/inner_join.groovy | 4 + .../mv/join/left_outer/outer_join.groovy | 11 + .../inner_join_infer_and_derive.groovy | 3 + .../left_join_infer_and_derive.groovy | 2 + .../right_join_infer_and_derive.groovy | 3 + .../mv/micro_test/micro_test_when_cte.groovy | 204 ++++++++ .../mv/negative/negative_test.groovy | 3 + .../mv/nested/nested_materialized_view.groovy | 4 + .../mv/nested_mtmv/nested_mtmv.groovy | 213 +++----- .../nested_mtmv_rewrite_switch.groovy | 3 + .../mv/partition_mv_rewrite.groovy | 4 + .../mv/scan/scan_table.groovy | 4 + .../union_all_compensate.groovy | 3 + .../partition_curd_union_rewrite.groovy | 3 + .../usercase_union_rewrite.groovy | 3 + .../unsafe_equals/null_un_safe_equals.groovy | 2 + .../mv/variant/variant_mv.groovy | 3 + .../with_auth/with_select_table_auth.groovy | 3 + .../query_with_sql_limit.groovy | 4 + .../with_table_operator.groovy | 195 ++++++++ .../mv/newMv/aggHaveDupBase.groovy | 1 + .../mv/newMv/case_ignore.groovy | 2 + .../mv/newMv/dup_gb_mv_abs.groovy | 1 + .../mv/newMv/dup_gb_mv_plus.groovy | 1 + .../mv/newMv/dup_mv_abs.groovy | 2 +- .../mv/newMv/dup_mv_bin.groovy | 1 + .../mv/newMv/dup_mv_bm_hash.groovy | 1 + .../mv/newMv/dup_mv_plus.groovy | 2 +- .../mv/newMv/dup_mv_year.groovy | 1 + .../mv/newMv/multi_slot1.groovy | 1 + .../mv/newMv/multi_slot2.groovy | 1 + .../mv/newMv/multi_slot3.groovy | 1 + .../mv/newMv/multi_slot4.groovy | 1 + .../mv/newMv/multi_slot5.groovy | 1 + .../mv/newMv/multi_slot6.groovy | 1 + .../mv/newMv/single_slot.groovy | 2 +- .../mv/newMv/sum_devide_count.groovy | 2 +- .../mv/newMv/unique_mv.groovy | 1 + .../mv/ut/MVMultiUsage.groovy | 3 + .../nereids_syntax_p0/mv/ut/MVWithAs.groovy | 3 + .../mv/ut/aggCDInBitmap.groovy | 9 + .../mv/ut/aggMVCalcAggFun.groovy | 2 +- .../mv/ut/aggOnAggMV1.groovy | 1 + .../mv/ut/aggOnAggMV10.groovy | 1 + .../mv/ut/aggOnAggMV11.groovy | 3 + .../mv/ut/aggOnAggMV2.groovy | 1 + .../mv/ut/aggOnAggMV3.groovy | 1 + .../mv/ut/aggOnAggMV5.groovy | 2 + .../mv/ut/aggOnAggMV6.groovy | 1 + .../mv/ut/aggOnAggMV7.groovy | 1 + .../mv/ut/bitmapUnionIn.groovy | 3 + .../mv/ut/incMVReInSub.groovy | 3 + .../mv/ut/incRewriteCD.groovy | 3 + .../mv/ut/joinOnLeftPToJoin.groovy | 5 + .../mv/ut/orderByOnPView.groovy | 2 + .../nereids_syntax_p0/mv/ut/projectMV1.groovy | 3 + .../nereids_syntax_p0/mv/ut/projectMV2.groovy | 3 + .../nereids_syntax_p0/mv/ut/projectMV3.groovy | 3 + .../nereids_syntax_p0/mv/ut/projectMV4.groovy | 3 + .../nereids_syntax_p0/mv/ut/subQuery.groovy | 2 +- .../nereids_syntax_p0/mv/ut/unionDis.groovy | 2 + .../nereids_syntax_p0/rollup/agg.groovy | 1 + .../nereids_syntax_p0/rollup/agg_date.groovy | 1 + .../nereids_syntax_p0/rollup/bitmap.groovy | 1 + .../nereids_syntax_p0/rollup/date.groovy | 1 + .../nereids_syntax_p0/rollup/hll/hll.groovy | 1 + .../hll_with_light_sc.groovy | 1 + 195 files changed, 2219 insertions(+), 480 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java create mode 100644 regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out create mode 100644 regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy create mode 100644 regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy create mode 100644 regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 660ebde293acbb..d3d7f1ad6ebbf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -51,14 +51,18 @@ public class MTMVCache { // The materialized view plan which should be optimized by the same rules to query // and will remove top sink and unused sort private final Plan logicalPlan; - // The original plan of mv def sql + // The original rewritten plan of mv def sql private final Plan originalPlan; + // The analyzed plan of mv def sql, which is used by tableCollector,should not be optimized by rbo + private final Plan analyzedPlan; private final Statistics statistics; private final StructInfo structInfo; - public MTMVCache(Plan logicalPlan, Plan originalPlan, Statistics statistics, StructInfo structInfo) { + public MTMVCache(Plan logicalPlan, Plan originalPlan, Plan analyzedPlan, + Statistics statistics, StructInfo structInfo) { this.logicalPlan = logicalPlan; this.originalPlan = originalPlan; + this.analyzedPlan = analyzedPlan; this.statistics = statistics; this.structInfo = structInfo; } @@ -71,6 +75,10 @@ public Plan getOriginalPlan() { return originalPlan; } + public Plan getAnalyzedPlan() { + return analyzedPlan; + } + public Statistics getStatistics() { return statistics; } @@ -118,7 +126,7 @@ public Plan visitLogicalResultSink(LogicalResultSink logicalResu Optional structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan, planner.getCascadesContext(), new BitSet()); - return new MTMVCache(mvPlan, originPlan, needCost + return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), needCost ? planner.getCascadesContext().getMemo().getRoot().getStatistics() : null, structInfoOptional.orElseGet(() -> null)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index eb47999117872e..5761b41171e644 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -35,12 +35,10 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; -import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.visitor.TableCollector; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -60,6 +58,12 @@ public static ConnectContext createMTMVContext(MTMV mtmv) { ctx.setThreadLocalInfo(); ctx.getSessionVariable().enableFallbackToOriginalPlanner = false; ctx.getSessionVariable().enableNereidsDML = true; + // Debug session variable should be disabled when refreshed + ctx.getSessionVariable().skipDeletePredicate = false; + ctx.getSessionVariable().skipDeleteBitmap = false; + ctx.getSessionVariable().skipDeleteSign = false; + ctx.getSessionVariable().skipStorageEngineMerge = false; + ctx.getSessionVariable().showHiddenColumns = false; ctx.getSessionVariable().allowModifyMaterializedViewData = true; // Disable add default limit rule to avoid refresh data wrong ctx.getSessionVariable().setDisableNereidsRules( @@ -100,31 +104,20 @@ private static void setCatalogAndDb(ConnectContext ctx, MTMV mtmv) { public static MTMVRelation generateMTMVRelation(MTMV mtmv, ConnectContext ctx) { // Should not make table without data to empty relation when analyze the related table, // so add disable rules - SessionVariable sessionVariable = ctx.getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - if (ctx.getStatementContext() != null) { - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - Plan plan; - try { - plan = getPlanBySql(mtmv.getQuerySql(), ctx); - } finally { - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - return generateMTMVRelation(plan); + Plan plan = getAnalyzePlanBySql(mtmv.getQuerySql(), ctx); + return generateMTMVRelation(plan, ctx); } - public static MTMVRelation generateMTMVRelation(Plan plan) { - return new MTMVRelation(getBaseTables(plan, true), getBaseTables(plan, false), getBaseViews(plan)); + public static MTMVRelation generateMTMVRelation(Plan plan, ConnectContext connectContext) { + return new MTMVRelation(getBaseTables(plan, true, connectContext), + getBaseTables(plan, false, connectContext), getBaseViews(plan)); } - private static Set getBaseTables(Plan plan, boolean expand) { + private static Set getBaseTables(Plan plan, boolean expand, ConnectContext connectContext) { TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( com.google.common.collect.Sets - .newHashSet(TableType.values()), expand); + .newHashSet(TableType.values()), expand, connectContext); plan.accept(TableCollector.INSTANCE, collectorContext); Set collectedTables = collectorContext.getCollectedTables(); return transferTableIfToInfo(collectedTables); @@ -142,7 +135,7 @@ private static Set transferTableIfToInfo(Set tables) { return result; } - private static Plan getPlanBySql(String querySql, ConnectContext ctx) { + private static Plan getAnalyzePlanBySql(String querySql, ConnectContext ctx) { List statements; try { statements = new NereidsParser().parseSQL(querySql); @@ -155,7 +148,7 @@ private static Plan getPlanBySql(String querySql, ConnectContext ctx) { ctx.setStatementContext(new StatementContext()); try { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.NONE); + return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN); } finally { ctx.setStatementContext(original); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java new file mode 100644 index 00000000000000..6d8d7c6326c0d9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.joinorder.hypergraph; + +/** + * This is the common base class for all + * */ +public interface HyperElement { + + // Get the references nodes + long getReferenceNodes(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java index 81a60b25031ca2..c9979ad002f702 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.jobs.joinorder.hypergraph.edge; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -32,7 +33,7 @@ /** * Edge in HyperGraph */ -public abstract class Edge { +public abstract class Edge implements HyperElement { private final int index; private final double selectivity; @@ -192,6 +193,7 @@ public boolean isSub(Edge edge) { return LongBitmap.isSubset(getReferenceNodes(), otherBitmap); } + @Override public long getReferenceNodes() { return LongBitmap.newBitmapUnion(leftExtendedNodes, rightExtendedNodes); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java index a4a64e0449deee..686576de771d94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.jobs.joinorder.hypergraph.node; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge; @@ -33,7 +34,7 @@ /** * HyperGraph Node. */ -public class AbstractNode { +public class AbstractNode implements HyperElement { protected final int index; protected final List joinEdges; protected final List filterEdges; @@ -65,6 +66,11 @@ public List getEdges() { .build(); } + @Override + public long getReferenceNodes() { + return getNodeMap(); + } + public int getIndex() { return index; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index e773d2721a9d6e..36363650cd099e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -237,6 +237,10 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, unboundRelation.getTableSample()); } } + if (!tabletIds.isEmpty()) { + // This tabletIds is set manually, so need to set specifiedTabletIds + scan = scan.withManuallySpecifiedTabletIds(tabletIds); + } if (needGenerateLogicalAggForRandomDistAggTable(scan)) { // it's a random distribution agg table // add agg on olap scan diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index e754e2f41281a9..ff1a5bdd09610b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -19,12 +19,14 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.jobs.joinorder.hypergraph.ConflictRulesMaker; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -51,6 +53,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -79,9 +82,9 @@ public class HyperGraphComparator { private final Map> pullUpViewExprWithEdge = new HashMap<>(); private final LogicalCompatibilityContext logicalCompatibilityContext; // this records the slots which needs to reject null - // the key is the target join which should reject null, the value is a pair, the first value of the pair is the - // join type, the second value is also a pair which left represents the slots in the left of join that should - // reject null, right represents the slots in the right of join that should reject null. + // the key is the view join edge which should reject null, the value is a pair, the first value of the pair is the + // query join type, the second value is also a pair which left represents the slots in the left of view join that + // should reject null, right represents the slots in the right of view join that should reject null. private final Map, Set>>> inferredViewEdgeWithCond = new HashMap<>(); private List viewJoinEdgesAfterInferring; private List viewFilterEdgesAfterInferring; @@ -249,9 +252,17 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) { } int size = queryExprSetList.size(); for (int i = 0; i < size; i++) { - Set mappingQueryExprSet = queryExprSetList.get(i).stream() - .map(logicalCompatibilityContext::getViewNodeExprFromQuery) - .collect(Collectors.toSet()); + Set queryExpressions = queryExprSetList.get(i); + Set mappingQueryExprSet = new HashSet<>(); + for (Expression queryExpression : queryExpressions) { + Optional mappingViewExprByQueryExpr = getMappingViewExprByQueryExpr(queryExpression, query, + this.logicalCompatibilityContext, + ExpressionPosition.NODE); + if (!mappingViewExprByQueryExpr.isPresent()) { + return false; + } + mappingQueryExprSet.add(mappingViewExprByQueryExpr.get()); + } if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) { return false; } @@ -407,7 +418,10 @@ private Map constructQueryToViewJoinMapWithExpr() { if (edgeMap.containsKey(entry.getValue())) { continue; } - Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey()); + Expression viewExpr = getMappingViewExprByQueryExpr(entry.getKey(), + entry.getValue(), + logicalCompatibilityContext, + ExpressionPosition.JOIN_EDGE).orElse(null); if (viewExprToEdge.containsKey(viewExpr)) { edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr))); } @@ -441,15 +455,19 @@ private Map constructQueryToViewFilterMapWithExpr() { HashMap queryToViewEdgeMap = new HashMap<>(); for (Entry> entry : queryExprToEdge.asMap().entrySet()) { - Expression queryExprViewBased = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey()); - if (queryExprViewBased == null) { - continue; - } - Collection viewEdges = viewExprToEdge.get(queryExprViewBased); - if (viewEdges.isEmpty()) { - continue; - } + Expression queryExprViewBased = null; for (Edge queryEdge : entry.getValue()) { + queryExprViewBased = getMappingViewExprByQueryExpr(entry.getKey(), + queryEdge, + logicalCompatibilityContext, + ExpressionPosition.FILTER_EDGE).orElse(null); + if (queryExprViewBased == null) { + continue; + } + Collection viewEdges = viewExprToEdge.get(queryExprViewBased); + if (viewEdges.isEmpty()) { + continue; + } for (Edge viewEdge : viewEdges) { if (!isSubTreeNodesEquals(queryEdge, viewEdge, logicalCompatibilityContext)) { // Such as query filter edge is <{1} --FILTER-- {}> but view filter edge is @@ -512,17 +530,17 @@ private boolean compareEdgeWithNode(Edge query, Edge view) { } private boolean compareFilterEdgeWithNode(FilterEdge query, FilterEdge view) { - return rewriteQueryNodeMap(query.getReferenceNodes()) == view.getReferenceNodes(); + return getViewNodesByQuery(query.getReferenceNodes()) == view.getReferenceNodes(); } private boolean compareJoinEdgeWithNode(JoinEdge query, JoinEdge view) { boolean res = false; if (query.getJoinType().swap() == view.getJoinType()) { - res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getRightExtendedNodes() - && rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getLeftExtendedNodes(); + res |= getViewNodesByQuery(query.getLeftExtendedNodes()) == view.getRightExtendedNodes() + && getViewNodesByQuery(query.getRightExtendedNodes()) == view.getLeftExtendedNodes(); } - res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes() - && rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getRightExtendedNodes(); + res |= getViewNodesByQuery(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes() + && getViewNodesByQuery(query.getRightExtendedNodes()) == view.getRightExtendedNodes(); return res; } @@ -545,8 +563,8 @@ private boolean compareJoinEdgeOrInfer(JoinEdge query, JoinEdge view) { } private boolean tryInferEdge(JoinEdge query, JoinEdge view) { - if (rewriteQueryNodeMap(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes() - || rewriteQueryNodeMap(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) { + if (getViewNodesByQuery(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes() + || getViewNodesByQuery(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) { return false; } if (!query.getJoinType().equals(view.getJoinType())) { @@ -567,7 +585,7 @@ private boolean tryInferEdge(JoinEdge query, JoinEdge view) { return true; } - private long rewriteQueryNodeMap(long bitmap) { + private long getViewNodesByQuery(long bitmap) { long newBitmap = LongBitmap.newBitmap(); for (int i : LongBitmap.getIterator(bitmap)) { int newIdx = getQueryToViewNodeIdMap().getOrDefault(i, 0); @@ -576,6 +594,35 @@ private long rewriteQueryNodeMap(long bitmap) { return newBitmap; } + private Optional getMappingViewExprByQueryExpr(Expression queryExpression, + HyperElement queryExpressionBelongedHyperElement, + LogicalCompatibilityContext context, + ExpressionPosition expressionPosition) { + Expression queryShuttledExpr; + Collection> viewExpressions; + if (ExpressionPosition.JOIN_EDGE.equals(expressionPosition)) { + queryShuttledExpr = context.getQueryJoinShuttledExpr(queryExpression); + viewExpressions = context.getViewJoinExprFromQuery(queryShuttledExpr); + } else if (ExpressionPosition.FILTER_EDGE.equals(expressionPosition)) { + queryShuttledExpr = context.getQueryFilterShuttledExpr(queryExpression); + viewExpressions = context.getViewFilterExprFromQuery(queryShuttledExpr); + } else { + queryShuttledExpr = context.getQueryNodeShuttledExpr(queryExpression); + viewExpressions = context.getViewNodeExprFromQuery(queryShuttledExpr); + } + if (viewExpressions.size() == 1) { + return Optional.of(viewExpressions.iterator().next().key()); + } + long queryReferenceNodes = queryExpressionBelongedHyperElement.getReferenceNodes(); + long viewReferenceNodes = getViewNodesByQuery(queryReferenceNodes); + for (Pair viewExpressionPair : viewExpressions) { + if (viewExpressionPair.value().getReferenceNodes() == viewReferenceNodes) { + return Optional.of(viewExpressionPair.key()); + } + } + return Optional.empty(); + } + private void compareJoinEdgeWithExpr(Edge query, Edge view) { Set queryExprSet = query.getExpressionSet(); Set viewExprSet = view.getExpressionSet(); @@ -583,7 +630,10 @@ private void compareJoinEdgeWithExpr(Edge query, Edge view) { Set exprMappedOfView = new HashSet<>(); List residualQueryExpr = new ArrayList<>(); for (Expression queryExpr : queryExprSet) { - Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr); + Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr, + query, + logicalCompatibilityContext, + ExpressionPosition.JOIN_EDGE).orElse(null); if (viewExprSet.contains(viewExpr)) { exprMappedOfView.add(viewExpr); } else { @@ -602,7 +652,10 @@ private void compareFilterEdgeWithExpr(Edge query, Edge view) { Set exprMappedOfView = new HashSet<>(); List residualQueryExpr = new ArrayList<>(); for (Expression queryExpr : queryExprSet) { - Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr); + Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr, + query, + logicalCompatibilityContext, + ExpressionPosition.FILTER_EDGE).orElse(null); if (viewExprSet.contains(viewExpr)) { exprMappedOfView.add(viewExpr); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 98866691ad87dd..925e8dba3c31fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -69,13 +69,18 @@ public void initMaterializationContext(CascadesContext cascadesContext) { * @param cascadesContext current cascadesContext in the planner */ protected void doInitMaterializationContext(CascadesContext cascadesContext) { + if (cascadesContext.getConnectContext().getSessionVariable().isInDebugMode()) { + LOG.info(String.format("MaterializationContext init return because is in debug mode, current queryId is %s", + cascadesContext.getConnectContext().getQueryIdentifier())); + return; + } // Only collect the table or mv which query use directly, to avoid useless mv partition in rewrite - TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false); + // Keep use one connection context when in query, if new connect context, + // the ConnectionContext.get() will change + TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false, + cascadesContext.getConnectContext()); try { Plan rewritePlan = cascadesContext.getRewritePlan(); - // Keep use one connection context when in query, if new connect context, - // the ConnectionContext.get() will change - collectorContext.setConnectContext(cascadesContext.getConnectContext()); rewritePlan.accept(TableCollector.INSTANCE, collectorContext); } catch (Exception e) { LOG.warn(String.format("MaterializationContext init table collect fail, current queryId is %s", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index 06098b591857ad..77ab37873d06b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -17,7 +17,10 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition; import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; @@ -27,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; @@ -34,8 +38,10 @@ import com.google.common.base.Suppliers; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; -import java.util.HashMap; +import java.util.Collection; import java.util.Map; import java.util.function.Supplier; @@ -45,9 +51,16 @@ public class LogicalCompatibilityContext { private final BiMap queryToViewNodeMapping; private final BiMap queryToViewNodeIDMapping; - private final Supplier> queryToViewJoinEdgeExpressionMappingSupplier; - private final Supplier> queryToViewNodeExpressionMappingSupplier; - private final Supplier> queryToViewFilterEdgeExpressionMappingSupplier; + private final ObjectId planNodeId; + private final Supplier>> + queryToViewJoinEdgeExpressionMappingSupplier; + private final Supplier> queryToQueryShuttledJoinExpressionMappingSupplier; + private final Supplier>> + queryToViewNodeExpressionMappingSupplier; + private final Supplier> queryToQueryShuttledNodeExpressionMappingSupplier; + private final Supplier>> + queryToViewFilterEdgeExpressionMappingSupplier; + private final Supplier> queryToQueryShuttledFilterExpressionMappingSupplier; /** * LogicalCompatibilityContext @@ -61,19 +74,31 @@ private LogicalCompatibilityContext(BiMap queryT queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE))); + this.queryToQueryShuttledJoinExpressionMappingSupplier = Suppliers.memoize( + () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.JOIN_EDGE)); + this.queryToViewNodeExpressionMappingSupplier = Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE))); + this.queryToQueryShuttledNodeExpressionMappingSupplier = Suppliers.memoize( + () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.NODE)); + this.queryToViewFilterEdgeExpressionMappingSupplier = Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE))); + this.queryToQueryShuttledFilterExpressionMappingSupplier = Suppliers.memoize( + () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.FILTER_EDGE)); + this.queryToViewNodeMapping = queryToViewNodeMapping; this.queryToViewNodeIDMapping = HashBiMap.create(); queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex())); + + this.planNodeId = queryStructInfo.getTopPlan().getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); } public BiMap getQueryToViewNodeMapping() { @@ -84,18 +109,30 @@ public BiMap getQueryToViewNodeIDMapping() { return queryToViewNodeIDMapping; } - public Expression getViewJoinExprFromQuery(Expression queryJoinExpr) { + public Collection> getViewJoinExprFromQuery(Expression queryJoinExpr) { return queryToViewJoinEdgeExpressionMappingSupplier.get().get(queryJoinExpr); } - public Expression getViewFilterExprFromQuery(Expression queryJoinExpr) { + public Expression getQueryJoinShuttledExpr(Expression queryJoinExpr) { + return queryToQueryShuttledJoinExpressionMappingSupplier.get().get(queryJoinExpr); + } + + public Collection> getViewFilterExprFromQuery(Expression queryJoinExpr) { return queryToViewFilterEdgeExpressionMappingSupplier.get().get(queryJoinExpr); } - public Expression getViewNodeExprFromQuery(Expression queryJoinExpr) { + public Expression getQueryFilterShuttledExpr(Expression queryFilterExpr) { + return queryToQueryShuttledFilterExpressionMappingSupplier.get().get(queryFilterExpr); + } + + public Collection> getViewNodeExprFromQuery(Expression queryJoinExpr) { return queryToViewNodeExpressionMappingSupplier.get().get(queryJoinExpr); } + public Expression getQueryNodeShuttledExpr(Expression queryNodeExpr) { + return queryToQueryShuttledNodeExpressionMappingSupplier.get().get(queryNodeExpr); + } + /** * Generate logical compatibility context, * this make expression mapping between query and view by relation and the slot in relation mapping @@ -126,24 +163,31 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping, viewStructInfo); } - private static BiMap generateExpressionMapping( + /** + * The result is multimap + * the key is shuttled query expr + * the value is original view expr collection + * */ + private static Multimap> generateExpressionMapping( Map viewToQuerySlotMapping, - Map queryShuttledExprToExprMap, - Map viewShuttledExprToExprMap) { - final Map viewEdgeToConjunctsMapQueryBased = new HashMap<>(); - BiMap queryToViewEdgeMapping = HashBiMap.create(); + Multimap> queryShuttledExprToExprMap, + Multimap> viewShuttledExprToExprMap) { + Multimap> queryToViewEdgeMapping = HashMultimap.create(); if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null || queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) { return queryToViewEdgeMapping; } + final Multimap> viewShuttledExprToExprMapQueryBased = + HashMultimap.create(); viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { - viewEdgeToConjunctsMapQueryBased.put( + viewShuttledExprToExprMapQueryBased.put( orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr); }); - queryShuttledExprToExprMap.forEach((exprSet, edge) -> { - Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet)); - if (viewExpr != null) { - queryToViewEdgeMapping.put(edge, viewExpr); + queryShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { + Collection> viewExpressions = viewShuttledExprToExprMapQueryBased.get( + orderSlotAsc(shuttledExpr)); + if (viewExpressions != null) { + queryToViewEdgeMapping.putAll(shuttledExpr, viewExpressions); } }); return queryToViewEdgeMapping; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index e1f9a4a6a6ac3f..f87f67d2e7a77c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -25,14 +25,21 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.StructInfoMap; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.BindRelation; import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.rewrite.EliminateSort; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -47,19 +54,25 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import com.google.common.collect.HashMultimap; @@ -299,7 +312,50 @@ public static List extractNondeterministicFunction(Plan plan) { return nondeterministicFunctions; } - private static final class TableQueryOperatorChecker extends DefaultPlanVisitor { + /** + * createMTMVCache from querySql + */ + public static MTMVCache createMTMVCache(String querySql, ConnectContext connectContext) { + LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(querySql); + StatementContext mvSqlStatementContext = new StatementContext(connectContext, + new OriginStatement(querySql, 0)); + NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); + if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) { + mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext); + } + // Can not convert to table sink, because use the same column from different table when self join + // the out slot is wrong + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainCommand.ExplainLevel.ALL_PLAN); + Plan originPlan = planner.getRewrittenPlan(); + // Eliminate result sink because sink operator is useless in query rewrite by materialized view + // and the top sort can also be removed + Plan mvPlan = originPlan.accept(new DefaultPlanRewriter() { + @Override + public Plan visitLogicalResultSink(LogicalResultSink logicalResultSink, Object context) { + return logicalResultSink.child().accept(this, context); + } + }, null); + // Optimize by rules to remove top sort + CascadesContext parentCascadesContext = CascadesContext.initContext(mvSqlStatementContext, mvPlan, + PhysicalProperties.ANY); + mvPlan = MaterializedViewUtils.rewriteByRules(parentCascadesContext, childContext -> { + Rewriter.getCteChildrenRewriter(childContext, + ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute(); + return childContext.getRewritePlan(); + }, mvPlan, originPlan); + return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), + planner.getCascadesContext().getMemo().getRoot().getStatistics(), null); + } + + /** + * Check the query if Contains query operator + * Such sql as following should return true + * select * from orders TABLET(10098) because TABLET(10098) should return true + * select * from orders_partition PARTITION (day_2) because PARTITION (day_2) + * select * from orders index query_index_test because index query_index_test + * select * from orders TABLESAMPLE(20 percent) because TABLESAMPLE(20 percent) + * */ + public static final class TableQueryOperatorChecker extends DefaultPlanVisitor { public static final TableQueryOperatorChecker INSTANCE = new TableQueryOperatorChecker(); @Override @@ -310,12 +366,20 @@ public Boolean visitLogicalRelation(LogicalRelation relation, Void context) { if (relation instanceof LogicalOlapScan) { LogicalOlapScan logicalOlapScan = (LogicalOlapScan) relation; if (logicalOlapScan.getTableSample().isPresent()) { + // Contain sample, select * from orders TABLESAMPLE(20 percent) return true; } - if (!logicalOlapScan.getSelectedTabletIds().isEmpty()) { + if (!logicalOlapScan.getManuallySpecifiedTabletIds().isEmpty()) { + // Contain tablets, select * from orders TABLET(10098) because TABLET(10098) return true; } if (!logicalOlapScan.getManuallySpecifiedPartitions().isEmpty()) { + // Contain specified partitions, select * from orders_partition PARTITION (day_2) + return true; + } + if (logicalOlapScan.getSelectedIndexId() != logicalOlapScan.getTable().getBaseIndexId()) { + // Contains select index or use sync mv in rbo rewrite + // select * from orders index query_index_test return true; } } @@ -492,6 +556,7 @@ public Void visitLogicalWindow(LogicalWindow window, IncrementCh @Override public Void visit(Plan plan, IncrementCheckerContext context) { if (plan instanceof LogicalProject + || plan instanceof LogicalLimit || plan instanceof LogicalFilter || plan instanceof LogicalJoin || plan instanceof LogicalAggregate diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index fa29d4d0e123be..28983a04f0e028 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -23,11 +23,12 @@ import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.executor.Rewriter; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; -import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.TableQueryOperatorChecker; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.copier.DeepCopierContext; import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; @@ -36,6 +37,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.ObjectId; @@ -61,12 +63,15 @@ import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.BitSet; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -109,9 +114,23 @@ public class StructInfo { // split predicates is shuttled private SplitPredicate splitPredicate; private EquivalenceClass equivalenceClass; - // Key is the expression shuttled and the value is the origin expression + // For value of Map, the key is the position of expression + // the value is the expressions and the hyper element of expression pair + // Key of pair is the expression shuttled and the value is the origin expression and the hyper element it belonged + // Sometimes origin expressions are different and shuttled expression is same + // Such as origin expressions are l_partkey#0 > 1 and l_partkey#10 > 1 and shuttled expression is l_partkey#10 > 1 // this is for building LogicalCompatibilityContext later. - private final Map> shuttledExpressionsToExpressionsMap; + private final Map>> + shuttledExpressionsToExpressionsMap; + // For value of Map, the key is the position of expression + // the value is the original expression and shuttled expression map + // Such as origin expressions are l_partkey#0 > 1 and shuttled expression is l_partkey#10 > 1 + // the map would be {ExpressionPosition.FILTER, { + // l_partkey#0 > 1 : l_partkey#10 > 1 + // }} + // this is for building LogicalCompatibilityContext later. + private final Map> expressionToShuttledExpressionToMap; + // Record the exprId and the corresponding expr map, this is used by expression shuttled private final Map namedExprIdAndExprMapping; private final List planOutputShuttledExpressions; @@ -123,7 +142,9 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG Plan bottomPlan, List relations, Map relationIdStructInfoNodeMap, @Nullable Predicates predicates, - Map> shuttledExpressionsToExpressionsMap, + Map>> + shuttledExpressionsToExpressionsMap, + Map> expressionToShuttledExpressionToMap, Map namedExprIdAndExprMapping, BitSet tableIdSet, SplitPredicate splitPredicate, @@ -142,6 +163,7 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG this.splitPredicate = splitPredicate; this.equivalenceClass = equivalenceClass; this.shuttledExpressionsToExpressionsMap = shuttledExpressionsToExpressionsMap; + this.expressionToShuttledExpressionToMap = expressionToShuttledExpressionToMap; this.namedExprIdAndExprMapping = namedExprIdAndExprMapping; this.planOutputShuttledExpressions = planOutputShuttledExpressions; } @@ -152,7 +174,8 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG public StructInfo withPredicates(Predicates predicates) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, - this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, this.tableBitSet, + this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, + this.namedExprIdAndExprMapping, this.tableBitSet, null, null, this.planOutputShuttledExpressions); } @@ -162,13 +185,16 @@ public StructInfo withPredicates(Predicates predicates) { public StructInfo withTableBitSet(BitSet tableBitSet) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, this.predicates, - this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, tableBitSet, + this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, + this.namedExprIdAndExprMapping, tableBitSet, this.splitPredicate, this.equivalenceClass, this.planOutputShuttledExpressions); } private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, Plan topPlan, - Map> shuttledExpressionsToExpressionsMap, + Map>> + shuttledExpressionsToExpressionsMap, + Map> expressionToShuttledExpressionToMap, Map namedExprIdAndExprMapping, List relations, Map relationIdStructInfoNodeMap, @@ -196,8 +222,9 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map List replacedExpressions = replaceContext.getReplacedExpressions(); - putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, - ExpressionPosition.NODE, replacedExpressions.get(0), expression); + putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, + expressionToShuttledExpressionToMap, + ExpressionPosition.NODE, replacedExpressions.get(0), expression, node); // Record this, will be used in top level expression shuttle later, see the method // ExpressionLineageReplacer#visitGroupPlan namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); @@ -210,28 +237,27 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, }); // Collect expression from join condition in hyper graph for (JoinEdge edge : hyperGraph.getJoinEdges()) { - List hashJoinConjuncts = edge.getHashJoinConjuncts(); + List joinConjunctExpressions = edge.getExpressions(); // shuttle expression in edge for the build of LogicalCompatibilityContext later. // Record the exprId to expr map in the processing to strut info // TODO get exprId to expr map when complex project is ready in join dege - hashJoinConjuncts.forEach(conjunctExpr -> { - ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = - new ExpressionLineageReplacer.ExpressionReplaceContext( - Lists.newArrayList(conjunctExpr), ImmutableSet.of(), - ImmutableSet.of(), new BitSet()); - topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); - // Replace expressions by expression map - List replacedExpressions = replaceContext.getReplacedExpressions(); - putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, - ExpressionPosition.JOIN_EDGE, replacedExpressions.get(0), conjunctExpr); - // Record this, will be used in top level expression shuttle later, see the method - // ExpressionLineageReplacer#visitGroupPlan - namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); - }); - List otherJoinConjuncts = edge.getOtherJoinConjuncts(); - if (!otherJoinConjuncts.isEmpty()) { - return false; + ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = + new ExpressionLineageReplacer.ExpressionReplaceContext( + joinConjunctExpressions.stream().map(expr -> (Expression) expr) + .collect(Collectors.toList()), + ImmutableSet.of(), ImmutableSet.of(), new BitSet()); + topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); + // Replace expressions by expression map + List replacedExpressions = replaceContext.getReplacedExpressions(); + for (int i = 0; i < replacedExpressions.size(); i++) { + putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, + expressionToShuttledExpressionToMap, + ExpressionPosition.JOIN_EDGE, replacedExpressions.get(i), joinConjunctExpressions.get(i), + edge); } + // Record this, will be used in top level expression shuttle later, see the method + // ExpressionLineageReplacer#visitGroupPlan + namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); } // Collect expression from where in hyper graph hyperGraph.getFilterEdges().forEach(filterEdge -> { @@ -239,10 +265,11 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, filterExpressions.forEach(predicate -> { // this is used for LogicalCompatibilityContext ExpressionUtils.extractConjunction(predicate).forEach(expr -> - putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, + putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, + expressionToShuttledExpressionToMap, ExpressionPosition.FILTER_EDGE, ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan, new BitSet()), - predicate)); + predicate, filterEdge)); }); }); return true; @@ -314,11 +341,13 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable // collect struct info fromGraph List relationList = new ArrayList<>(); Map relationIdStructInfoNodeMap = new LinkedHashMap<>(); - Map> shuttledHashConjunctsToConjunctsMap = - new LinkedHashMap<>(); + Map>> + shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>(); Map namedExprIdAndExprMapping = new LinkedHashMap<>(); BitSet tableBitSet = new BitSet(); + Map> expressionToShuttledExpressionToMap = new HashMap<>(); boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap, + expressionToShuttledExpressionToMap, namedExprIdAndExprMapping, relationList, relationIdStructInfoNodeMap, @@ -326,6 +355,11 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable cascadesContext); valid = valid && hyperGraph.getNodes().stream().allMatch(n -> ((StructInfoNode) n).getExpressions() != null); + // if relationList has any relation which contains table operator, + // such as query with sample, index, table, is invalid + boolean invalid = relationList.stream().anyMatch(relation -> + ((AbstractPlan) relation).accept(TableQueryOperatorChecker.INSTANCE, null)); + valid = valid && !invalid; // collect predicate from top plan which not in hyper graph Set topPlanPredicates = new LinkedHashSet<>(); topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); @@ -335,19 +369,11 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable ExpressionUtils.shuttleExpressionWithLineage(originalPlan.getOutput(), originalPlan, new BitSet()); return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, relationList, relationIdStructInfoNodeMap, predicates, shuttledHashConjunctsToConjunctsMap, + expressionToShuttledExpressionToMap, namedExprIdAndExprMapping, tableBitSet, null, null, planOutputShuttledExpressions); } - /** - * Build Struct info from group. - * Maybe return multi structInfo when original plan already be rewritten by mv - */ - public static StructInfo of(Group group) { - // TODO build graph from original plan and get relations and predicates from graph - return null; - } - public List getRelations() { return relations; } @@ -404,21 +430,36 @@ public Map getRelationIdStructInfoNodeMap() { return relationIdStructInfoNodeMap; } - public Map> getShuttledExpressionsToExpressionsMap() { + public Map>> + getShuttledExpressionsToExpressionsMap() { return shuttledExpressionsToExpressionsMap; } - private static void putShuttledExpressionsToExpressionsMap( - Map> shuttledExpressionsToExpressionsMap, + public Map> getExpressionToShuttledExpressionToMap() { + return expressionToShuttledExpressionToMap; + } + + private static void putShuttledExpressionToExpressionsMap( + Map>> + shuttledExpressionsToExpressionsMap, + Map> expressionPositionToExpressionToMap, ExpressionPosition expressionPosition, - Expression key, Expression value) { - Map expressionExpressionMap = shuttledExpressionsToExpressionsMap.get( - expressionPosition); - if (expressionExpressionMap == null) { - expressionExpressionMap = new LinkedHashMap<>(); - shuttledExpressionsToExpressionsMap.put(expressionPosition, expressionExpressionMap); - } - expressionExpressionMap.put(key, value); + Expression shuttledExpression, Expression originalExpression, HyperElement valueBelongedElement) { + Multimap> shuttledExpressionToExpressionMap = + shuttledExpressionsToExpressionsMap.get(expressionPosition); + if (shuttledExpressionToExpressionMap == null) { + shuttledExpressionToExpressionMap = HashMultimap.create(); + shuttledExpressionsToExpressionsMap.put(expressionPosition, shuttledExpressionToExpressionMap); + } + shuttledExpressionToExpressionMap.put(shuttledExpression, Pair.of(originalExpression, valueBelongedElement)); + + Map originalExprToShuttledExprMap = + expressionPositionToExpressionToMap.get(expressionPosition); + if (originalExprToShuttledExprMap == null) { + originalExprToShuttledExprMap = new HashMap<>(); + expressionPositionToExpressionToMap.put(expressionPosition, originalExprToShuttledExprMap); + } + originalExprToShuttledExprMap.put(originalExpression, shuttledExpression); } public List getExpressions() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java index 5465211c73e793..c7b8f452afbe66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java @@ -36,11 +36,16 @@ public Rule build() { return logicalOlapScan().thenApply(ctx -> { LogicalOlapScan scan = ctx.root; OlapTable table = scan.getTable(); - List ids = table.selectNonEmptyPartitionIds(scan.getSelectedPartitionIds()); + List partitionIdsToPrune = scan.getSelectedPartitionIds(); + List ids = table.selectNonEmptyPartitionIds(partitionIdsToPrune); if (ids.isEmpty()) { return new LogicalEmptyRelation(ConnectContext.get().getStatementContext().getNextRelationId(), scan.getOutput()); } + if (partitionIdsToPrune.equals(ids)) { + // Not Prune actually, return directly + return null; + } return scan.withSelectedPartitionIds(ids); }).toRule(RuleType.PRUNE_EMPTY_PARTITION); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java index 4b86b7a3706f0e..fc212550b6f364 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java @@ -53,7 +53,7 @@ public Rule build() { LogicalOlapScan olapScan = filter.child(); OlapTable table = olapScan.getTable(); Builder selectedTabletIdsBuilder = ImmutableList.builder(); - if (olapScan.getSelectedTabletIds().isEmpty()) { + if (olapScan.getManuallySpecifiedTabletIds().isEmpty()) { for (Long id : olapScan.getSelectedPartitionIds()) { Partition partition = table.getPartition(id); MaterializedIndex index = partition.getIndex(olapScan.getSelectedIndexId()); @@ -64,10 +64,10 @@ public Rule build() { partition.getDistributionInfo())); } } else { - selectedTabletIdsBuilder.addAll(olapScan.getSelectedTabletIds()); + selectedTabletIdsBuilder.addAll(olapScan.getManuallySpecifiedTabletIds()); } List selectedTabletIds = selectedTabletIdsBuilder.build(); - if (new HashSet<>(selectedTabletIds).equals(new HashSet<>(olapScan.getSelectedTabletIds()))) { + if (new HashSet<>(selectedTabletIds).equals(new HashSet<>(olapScan.getManuallySpecifiedTabletIds()))) { return null; } return filter.withChildren(olapScan.withSelectedTabletIds(selectedTabletIds)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index de284bd837748f..36cc0f95a77a8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -246,11 +246,16 @@ public Plan visitLogicalCTE(LogicalCTE cte, PredicateAddContext if (predicates.isEmpty()) { return cte; } + List> rewrittenSubQueryAlias = new ArrayList<>(); for (LogicalSubQueryAlias subQueryAlias : cte.getAliasQueries()) { + List subQueryAliasChildren = new ArrayList<>(); this.virtualRelationNamePartSet.add(subQueryAlias.getQualifier()); - subQueryAlias.children().forEach(subQuery -> subQuery.accept(this, predicates)); + subQueryAlias.children().forEach(subQuery -> + subQueryAliasChildren.add(subQuery.accept(this, predicates)) + ); + rewrittenSubQueryAlias.add(subQueryAlias.withChildren(subQueryAliasChildren)); } - return super.visitLogicalCTE(cte, predicates); + return super.visitLogicalCTE(new LogicalCTE<>(rewrittenSubQueryAlias, cte.child()), predicates); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 837d9478c89327..867b22dcde746e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -156,6 +156,9 @@ public void analyze(ConnectContext ctx) throws Exception { if (!InternalCatalog.INTERNAL_CATALOG_NAME.equals(mvName.getCtl())) { throw new AnalysisException("Only support creating asynchronous materialized views in internal catalog"); } + if (ctx.getSessionVariable().isInDebugMode()) { + throw new AnalysisException("Create materialized view fail, because is in debug mode"); + } try { FeNameFormat.checkTableName(mvName.getTbl()); } catch (org.apache.doris.common.AnalysisException e) { @@ -252,9 +255,21 @@ public void analyzeQuery(ConnectContext ctx, Map mvProperties) t NereidsPlanner planner = new NereidsPlanner(statementContext); // this is for expression column name infer when not use alias LogicalSink logicalSink = new UnboundResultSink<>(logicalQuery); - // must disable constant folding by be, because be constant folding may return wrong type - ctx.getSessionVariable().disableConstantFoldingByBEOnce(); - Plan plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + // Should not make table without data to empty relation when analyze the related table, + // so add disable rules + Set tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames(); + ctx.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + Plan plan; + try { + // must disable constant folding by be, because be constant folding may return wrong type + ctx.getSessionVariable().disableConstantFoldingByBEOnce(); + plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + } finally { + // after operate, roll back the disable rules + ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules)); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + } // can not contain VIEW or MTMV analyzeBaseTables(planner.getAnalyzedPlan()); // can not contain Random function @@ -265,8 +280,7 @@ public void analyzeQuery(ConnectContext ctx, Map mvProperties) t throw new AnalysisException("can not contain invalid expression"); } getRelation(planner); - this.mvPartitionInfo = mvPartitionDefinition - .analyzeAndTransferToMTMVPartitionInfo(planner, ctx, logicalQuery); + this.mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner, ctx); this.partitionDesc = generatePartitionDesc(ctx); getColumns(plan, ctx, mvPartitionInfo.getPartitionCol(), distribution); analyzeKeys(); @@ -311,24 +325,10 @@ private void analyzeKeys() { } } + // Should use analyzed plan for collect views and tables private void getRelation(NereidsPlanner planner) { - // Should not make table without data to empty relation when analyze the related table, - // so add disable rules - ConnectContext ctx = planner.getCascadesContext().getConnectContext(); - SessionVariable sessionVariable = ctx.getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - if (ctx.getStatementContext() != null) { - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - Plan plan; - try { - plan = planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.NONE); - } finally { - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - this.relation = MTMVPlanUtil.generateMTMVRelation(plan); + this.relation = MTMVPlanUtil.generateMTMVRelation(planner.getAnalyzedPlan(), + planner.getCascadesContext().getConnectContext()); } private PartitionDesc generatePartitionDesc(ConnectContext ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java index c4117e8608e29d..a26a97f7240793 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java @@ -37,7 +37,6 @@ import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; import org.apache.doris.nereids.trees.expressions.Cast; @@ -45,11 +44,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import com.google.common.collect.Sets; @@ -72,11 +67,9 @@ public class MTMVPartitionDefinition { * * @param planner planner * @param ctx ctx - * @param logicalQuery logicalQuery * @return MTMVPartitionInfo */ - public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx, - LogicalPlan logicalQuery) { + public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx) { MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(partitionType); if (this.partitionType == MTMVPartitionType.SELF_MANAGE) { return mtmvPartitionInfo; @@ -100,7 +93,7 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl timeUnit = null; } mtmvPartitionInfo.setPartitionCol(partitionColName); - RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName, timeUnit); + RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, partitionColName, timeUnit); mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn()); mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo()); if (relatedTableInfo.getPartitionExpression().isPresent()) { @@ -125,47 +118,33 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl return mtmvPartitionInfo; } - private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, LogicalPlan - logicalQuery, - String partitionColName, - String timeUnit) { + // Should use rewritten plan without view and subQuery to get related partition table + private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, + String partitionColName, String timeUnit) { CascadesContext cascadesContext = planner.getCascadesContext(); - SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - // Should not make table without data to empty relation when analyze the related table, - // so add disable rules - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + + RelatedTableInfo relatedTableInfo = MaterializedViewUtils + .getRelatedTableInfo(partitionColName, timeUnit, planner.getRewrittenPlan(), cascadesContext); + if (!relatedTableInfo.isPctPossible()) { + throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning," + + " the fail reason is %s", relatedTableInfo.getFailReason())); + } + MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo()); + Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); try { - Plan mvRewrittenPlan = - planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); - RelatedTableInfo relatedTableInfo = MaterializedViewUtils - .getRelatedTableInfo(partitionColName, timeUnit, mvRewrittenPlan, cascadesContext); - if (!relatedTableInfo.isPctPossible()) { - throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning," - + " the fail reason is %s", relatedTableInfo.getFailReason())); - } - MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo()); - Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - try { - partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames(Optional.empty())); - } catch (DdlException e) { - throw new AnalysisException(e.getMessage(), e); - } + partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames(Optional.empty())); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage(), e); + } - if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) { - throw new AnalysisException("error related column: " + relatedTableInfo.getColumn()); - } - if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) - && partitionColumnNames.size() != 1) { - throw new AnalysisException("only hms table support multi column partition."); - } - return relatedTableInfo; - } finally { - // after operate, roll back the disable rules - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) { + throw new AnalysisException("error related column: " + relatedTableInfo.getColumn()); + } + if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) + && partitionColumnNames.size() != 1) { + throw new AnalysisException("only hms table support multi column partition."); } + return relatedTableInfo; } private static List convertToLegacyArguments(List children) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 2f0b3b0a769cb1..ca4619dd00dac2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -92,6 +92,12 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan */ private final List selectedTabletIds; + /** + * Selected tablet ids to read data from, this would be set if user query with tablets manually + * Such as select * from orders TABLET(100); + */ + private final List manuallySpecifiedTabletIds; + /////////////////////////////////////////////////////////////////////////// // Members for partition ids. /////////////////////////////////////////////////////////////////////////// @@ -120,12 +126,16 @@ public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } + /** + * LogicalOlapScan construct method + */ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, ImmutableList.of(), -1, false, PreAggStatus.unset(), ImmutableList.of(), ImmutableList.of(), - Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of()); + Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of(), + ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List tabletIds, @@ -133,7 +143,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, L this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, tabletIds, -1, false, PreAggStatus.unset(), ImmutableList.of(), hints, Maps.newHashMap(), - tableSample, false, ImmutableMap.of()); + tableSample, false, ImmutableMap.of(), ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List specifiedPartitions, @@ -142,7 +152,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, L // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1' specifiedPartitions, false, tabletIds, -1, false, PreAggStatus.unset(), specifiedPartitions, hints, Maps.newHashMap(), - tableSample, false, ImmutableMap.of()); + tableSample, false, ImmutableMap.of(), ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List tabletIds, @@ -151,7 +161,8 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, L this(id, table, qualifier, Optional.empty(), Optional.empty(), selectedPartitionIds, false, tabletIds, selectedIndexId, true, preAggStatus, - specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of()); + specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of(), + ImmutableList.of()); } /** @@ -164,7 +175,7 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, PreAggStatus preAggStatus, List specifiedPartitions, List hints, Map, Slot> cacheSlotWithSlotName, Optional tableSample, boolean directMvScan, - Map>> colToSubPathsMap) { + Map>> colToSubPathsMap, List specifiedTabletIds) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties); Preconditions.checkArgument(selectedPartitionIds != null, @@ -175,6 +186,7 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, this.indexSelected = indexSelected; this.preAggStatus = preAggStatus; this.manuallySpecifiedPartitions = ImmutableList.copyOf(specifiedPartitions); + this.manuallySpecifiedTabletIds = ImmutableList.copyOf(specifiedTabletIds); if (selectedPartitionIds.isEmpty()) { this.selectedPartitionIds = ImmutableList.of(); @@ -233,6 +245,7 @@ public boolean equals(Object o) { && partitionPruned == that.partitionPruned && Objects.equals(preAggStatus, that.preAggStatus) && Objects.equals(selectedTabletIds, that.selectedTabletIds) && Objects.equals(manuallySpecifiedPartitions, that.manuallySpecifiedPartitions) + && Objects.equals(manuallySpecifiedTabletIds, that.manuallySpecifiedTabletIds) && Objects.equals(selectedPartitionIds, that.selectedPartitionIds) && Objects.equals(hints, that.hints) && Objects.equals(tableSample, that.tableSample); @@ -241,8 +254,8 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash(super.hashCode(), selectedIndexId, indexSelected, preAggStatus, cacheSlotWithSlotName, - selectedTabletIds, partitionPruned, manuallySpecifiedPartitions, selectedPartitionIds, hints, - tableSample); + selectedTabletIds, partitionPruned, manuallySpecifiedTabletIds, manuallySpecifiedPartitions, + selectedPartitionIds, hints, tableSample); } @Override @@ -251,7 +264,7 @@ public LogicalOlapScan withGroupExpression(Optional groupExpres groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } @Override @@ -260,7 +273,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr return new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { @@ -268,7 +281,7 @@ public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withMaterializedIndexSelected(long indexId) { @@ -276,7 +289,7 @@ public LogicalOlapScan withMaterializedIndexSelected(long indexId) { Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, indexId, true, PreAggStatus.unset(), manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, - tableSample, directMvScan, colToSubPathsMap); + tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { @@ -284,7 +297,7 @@ public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { @@ -292,7 +305,7 @@ public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withColToSubPathsMap(Map>> colToSubPathsMap) { @@ -300,7 +313,15 @@ public LogicalOlapScan withColToSubPathsMap(Map>> colTo Optional.empty(), Optional.empty(), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); + } + + public LogicalOlapScan withManuallySpecifiedTabletIds(List manuallySpecifiedTabletIds) { + return new LogicalOlapScan(relationId, (Table) table, qualifier, + Optional.empty(), Optional.of(getLogicalProperties()), + selectedPartitionIds, partitionPruned, selectedTabletIds, + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } @Override @@ -310,7 +331,7 @@ public LogicalOlapScan withRelationId(RelationId relationId) { Optional.empty(), Optional.empty(), selectedPartitionIds, false, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap); + hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap, selectedTabletIds); } @Override @@ -326,6 +347,10 @@ public List getSelectedTabletIds() { return selectedTabletIds; } + public List getManuallySpecifiedTabletIds() { + return manuallySpecifiedTabletIds; + } + @Override public long getSelectedIndexId() { return selectedIndexId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index 2e2cdb810f0f72..27ff1e4b68c075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -20,8 +20,8 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.MTMVCache; -import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; @@ -70,13 +70,19 @@ public Plan visitPhysicalCatalogRelation(PhysicalCatalogRelation catalogRelation } private void expandMvAndCollect(MTMV mtmv, TableCollectorContext context) { - if (!context.isExpand()) { + if (!context.isExpandMaterializedView()) { return; } // Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong - MTMVCache expandedMv = MTMVCache.from(mtmv, context.getConnectContext() == null - ? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext(), false); - expandedMv.getLogicalPlan().accept(this, context); + MTMVCache expandedMvCache; + try { + expandedMvCache = mtmv.getOrGenerateCache(context.getConnectContext()); + } catch (AnalysisException exception) { + LOG.warn(String.format("expandMvAndCollect getOrGenerateCache fail, mtmv name is %s", mtmv.getName()), + exception); + expandedMvCache = MTMVCache.from(mtmv, context.getConnectContext(), false); + } + expandedMvCache.getAnalyzedPlan().accept(this, context); } /** @@ -87,12 +93,14 @@ public static final class TableCollectorContext { private final Set collectedTables = new HashSet<>(); private final Set targetTableTypes; // if expand the mv or not - private final boolean expand; - private ConnectContext connectContext; + private final boolean expandMaterializedView; + private final ConnectContext connectContext; - public TableCollectorContext(Set targetTableTypes, boolean expand) { + public TableCollectorContext(Set targetTableTypes, boolean expandMaterializedView, + ConnectContext connectContext) { this.targetTableTypes = targetTableTypes; - this.expand = expand; + this.expandMaterializedView = expandMaterializedView; + this.connectContext = connectContext; } public Set getCollectedTables() { @@ -103,16 +111,12 @@ public Set getTargetTableTypes() { return targetTableTypes; } - public boolean isExpand() { - return expand; + public boolean isExpandMaterializedView() { + return expandMaterializedView; } public ConnectContext getConnectContext() { return connectContext; } - - public void setConnectContext(ConnectContext connectContext) { - this.connectContext = connectContext; - } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java index e7940cdfb9b519..385657cd3f229e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; import org.apache.doris.nereids.trees.expressions.InPredicate; @@ -51,8 +52,9 @@ import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Objects; -class PruneOlapScanTabletTest implements MemoPatternMatchSupported { +class PruneOlapScanTabletTest extends SqlTestBase implements MemoPatternMatchSupported { @Test void testPruneOlapScanTablet(@Mocked OlapTable olapTable, @@ -154,4 +156,21 @@ void testPruneOlapScanTablet(@Mocked OlapTable olapTable, ) ); } + + @Test + void testPruneOlapScanTabletWithManually() { + String sql = "select * from T4 TABLET(110) where id > 8"; + PlanChecker.from(connectContext) + .analyze(sql) + .applyTopDown(new PruneOlapScanTablet()) + .matches( + logicalFilter( + logicalOlapScan().when(s -> + Objects.equals(s.getSelectedTabletIds(), Lists.newArrayList(110L)) + && Objects.equals(s.getManuallySpecifiedTabletIds(), + Lists.newArrayList(110L)) + ) + ) + ); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java index f5b301dcc49b25..9ebbc22a85c826 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java @@ -87,6 +87,9 @@ protected void runBeforeAll() throws Exception { + " score bigint\n" + ")\n" + "DUPLICATE KEY(id)\n" + + "AUTO PARTITION BY LIST(`id`)\n" + + "(\n" + + ")\n" + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + "PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java index 5fbc6d7b7ac254..9c56bb1186a5ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java @@ -130,7 +130,7 @@ public void test1() { Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Random); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -159,7 +159,7 @@ public void test2() { Assertions.assertTrue(nondeterministicFunctionSet.get(1) instanceof Random); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -196,7 +196,7 @@ public BitSet getDisableNereidsRules() { Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Uuid); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -210,7 +210,7 @@ public BitSet getDisableNereidsRules() { TableCollectorContext collectorContextWithNoExpand = new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP), - false); + false, connectContext); physicalPlan.accept(TableCollector.INSTANCE, collectorContextWithNoExpand); Set expectedTablesWithNoExpand = new HashSet<>(); expectedTablesWithNoExpand.add("table1"); @@ -222,7 +222,7 @@ public BitSet getDisableNereidsRules() { expectedTablesWithNoExpand); TableCollectorContext mvCollectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.MATERIALIZED_VIEW), true); + Sets.newHashSet(TableType.MATERIALIZED_VIEW), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContext); Set expectedMvs = new HashSet<>(); expectedMvs.add("mv1"); @@ -234,7 +234,7 @@ public BitSet getDisableNereidsRules() { TableCollectorContext mvCollectorContextWithNoExpand = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.MATERIALIZED_VIEW), false); + Sets.newHashSet(TableType.MATERIALIZED_VIEW), false, connectContext); physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContextWithNoExpand); Set expectedMvsWithNoExpand = new HashSet<>(); expectedMvsWithNoExpand.add("mv1"); @@ -246,7 +246,7 @@ public BitSet getDisableNereidsRules() { TableCollectorContext allTableTypeWithExpand = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.values()), true); + Sets.newHashSet(TableType.values()), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, allTableTypeWithExpand); // when collect in plan with expand, should collect table which is expended Set expectedTablesWithExpand = new HashSet<>(); diff --git a/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out b/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out index 1f27d180c736ce..c00bbc9377409e 100644 --- a/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out +++ b/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out @@ -1,11 +1,18 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container +19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container +19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container 19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container -- !select_mv -- -4 +16 -- !select -- -4 +16 diff --git a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out index 5c9df6b7f92256..c400e078daff1a 100644 --- a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out +++ b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out @@ -323,3 +323,27 @@ c 3 6 c,c,c 5.333333333333333 mi 3 2 1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 +-- !query29_0_before -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query29_0_after -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query30_0_before -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query30_0_after -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query31_0_before -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query31_0_after -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + diff --git a/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out b/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out new file mode 100644 index 00000000000000..8abaaf9adec0a8 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out @@ -0,0 +1,128 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query_0_after -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_0 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_1 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_0_after -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_2 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_3 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index fa43c3852778f2..fe355e01f7753e 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1515,6 +1515,22 @@ class Suite implements GroovyInterceptable { sql "analyze table ${db}.${mv_name} with sync;" } + def create_async_partition_mv = { db, mv_name, mv_sql, partition_col -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${db}.${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${db}.${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY ${partition_col} + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + sql "analyze table ${db}.${mv_name} with sync;" + } + // mv not part in rewrite process def mv_not_part_in = { query_sql, mv_name -> logger.info("query_sql = " + query_sql + ", mv_names = " + mv_name) @@ -1601,9 +1617,8 @@ class Suite implements GroovyInterceptable { check { result -> boolean success = true; for (String mv_name : mv_names) { - success = success && result.contains("${mv_name} chose") + Assert.assertEquals(true, result.contains("${mv_name} chose")) } - Assert.assertEquals(true, success) } } } diff --git a/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy b/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy index d9f73bcf89f567..d354b4b0e17ff5 100644 --- a/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy +++ b/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy @@ -56,6 +56,8 @@ suite ("agg_have_dup_base") { qt_select_mv "select unix_timestamp(k1) tmp,sum(k2) from d_table group by tmp order by tmp;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='5');""" + mv_rewrite_success("select k1,sum(k2),max(k2) from d_table group by k1;", "k12s3m") mv_rewrite_success("select k1,sum(k2) from d_table group by k1;", "k12s3m") diff --git a/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy b/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy index 1e34c70ccec900..29190396230dce 100644 --- a/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy +++ b/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy @@ -69,6 +69,7 @@ suite ("test_agg_state_max_by") { qt_select_star "select * from d_table order by 1,2;" mv_rewrite_success("select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;", "k1mb") sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" mv_rewrite_success("select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;", "k1mb") qt_select_mv "select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;" @@ -101,6 +102,7 @@ suite ("test_agg_state_max_by") { qt_select_star "select * from d_table order by 1,2;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" sql "analyze table d_table with sync;" sql """set enable_stats=false;""" @@ -114,6 +116,7 @@ suite ("test_agg_state_max_by") { qt_select_mv "select k1,max_by(k2,abs(k3)) from d_table group by k1 order by 1,2;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" mv_rewrite_success("select k1,max_by(k2+k3,abs(k3)) from d_table group by k1 order by 1,2;", "k1mbcp1") mv_rewrite_success("select k1,max_by(k2+k3,k3) from d_table group by k1 order by 1,2;", "k1mbcp2") mv_rewrite_success("select k1,max_by(k2,abs(k3)) from d_table group by k1 order by 1,2;", "k1mbcp3") diff --git a/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy b/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy index 1134d930185b9a..271eb0cb5ed190 100644 --- a/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy +++ b/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy @@ -37,6 +37,8 @@ suite ("case_ignore") { sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 3,-3,null,'c';" + sql """alter table d_table modify column k4 set stats ('row_count'='4');""" + createMV ("create materialized view k12a as select K1,abs(K2) from d_table;") sql "insert into d_table select -4,-4,-4,'d';" @@ -53,6 +55,7 @@ suite ("case_ignore") { qt_select_mv "select K1,abs(K2) from d_table order by K1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" mv_rewrite_success("select k1,abs(k2) from d_table order by k1;", "k12a") mv_rewrite_success("select K1,abs(K2) from d_table order by K1;", "k12a") diff --git a/regression-test/suites/mv_p0/count_star/count_star.groovy b/regression-test/suites/mv_p0/count_star/count_star.groovy index 52edda58276ef6..4d951d76fa98a9 100644 --- a/regression-test/suites/mv_p0/count_star/count_star.groovy +++ b/regression-test/suites/mv_p0/count_star/count_star.groovy @@ -64,6 +64,8 @@ suite ("count_star") { qt_select_mv "select count(*) from d_table where k3=1;" sql """set enable_stats=true;""" + + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" mv_rewrite_success("select k1,k4,count(*) from d_table group by k1,k4;", "kstar") mv_rewrite_success("select k1,k4,count(*) from d_table where k1=1 group by k1,k4;", "kstar") mv_rewrite_fail("select k1,k4,count(*) from d_table where k3=1 group by k1,k4;", "kstar") diff --git a/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy b/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy index e30af70b0041b3..14ebb67ddc8168 100644 --- a/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy +++ b/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy @@ -28,6 +28,8 @@ suite ("dis_26495") { sql """insert into doris_test values (1,2,max_by_state(1,2));""" + sql """alter table doris_test modify column agg_st_1 set stats ('row_count'='1');""" + streamLoad { table "doris_test" set 'column_separator', ',' diff --git a/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy b/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy index cfe2a5307d7ed4..b4c2921669663a 100644 --- a/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy +++ b/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy @@ -52,6 +52,8 @@ suite ("k1ap2spa") { sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" + mv_rewrite_success("select abs(k1)+1 t,sum(abs(k2+1)) from d_table group by t order by t;", "k1ap2spa") } diff --git a/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy b/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy index 02617d68f6020b..6c257755596d17 100644 --- a/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy +++ b/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy @@ -36,6 +36,7 @@ suite ("k1s2m3") { sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 3,-3,null,'c';" + sql """alter table d_table modify column k1 set stats ('row_count'='6');""" createMV("create materialized view k1s2m3 as select k1,sum(k2*k3) from d_table group by k1;") sql "insert into d_table select -4,-4,-4,'d';" diff --git a/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy b/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy index 367cd9863c1e3e..17583741e68b91 100644 --- a/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy +++ b/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy @@ -57,5 +57,6 @@ suite ("k1s2m3_auto_inc") { qt_select_mv "select k3,sum(abs(k2+1)) from d_table group by k3 order by 1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='2');""" mv_rewrite_success("select k3,sum(abs(k2+1)) from d_table group by k3 order by 1;", "k3ap2spa") } diff --git a/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy b/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy index a1d254b333e211..538d050fec8a9d 100644 --- a/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy +++ b/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy @@ -64,6 +64,7 @@ suite ("multi_agg_with_same_slot") { sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k2,avg(k3),max(k3) from d_table group by k1,k2 order by 1,2;", "kmv") mv_rewrite_success("select k1,k2,avg(k3)+max(k3) from d_table group by k1,k2 order by 1,2;", "kmv") mv_rewrite_success("select k1,k2,avg(k3)+max(k3) from d_table group by grouping sets((k1),(k1,k2),()) order by 1,2;", "kmv") diff --git a/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy b/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy index 761257230e245d..d06645e00cdfe0 100644 --- a/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy @@ -66,5 +66,6 @@ suite ("multi_slot_k123p") { qt_select_mv "select k1,version() from d_table order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k2+k3 from d_table order by k1;", "k123p") } diff --git a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy index e64adbb06b2988..fc722c5a07fbe7 100644 --- a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy @@ -51,6 +51,7 @@ suite ("multi_slot_k1a2p2ap3p") { qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table order by abs(k1)+k2+1,abs(k2+2)+k3+3;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='7');""" mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p") } diff --git a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy index 3a2f58f38d65a9..b2292084504199 100644 --- a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy @@ -61,6 +61,7 @@ suite ("multi_slot_k1a2p2ap3ps") { qt_select_base "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2 order by abs(k1)+k2;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1", "k1a2p2ap3ps") mv_rewrite_fail("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2 order by abs(k1)+k2", "k1a2p2ap3ps") diff --git a/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy b/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy index fa67fd3889d93e..75bbd3fb4b7bb8 100644 --- a/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy @@ -50,5 +50,6 @@ suite ("multi_slot_k1p2ap3p") { qt_select_mv "select k1+1,abs(k2+2)+k3+3 from d_table order by k1+1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1+1,abs(k2+2)+k3+3 from d_table order by k1+1;", "k1p2ap3p") } diff --git a/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy b/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy index e2f3564e2de2cb..4ef8a746a2b797 100644 --- a/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy @@ -49,5 +49,6 @@ suite ("multi_slot_k1p2ap3ps") { qt_select_mv "select k1+1,sum(abs(k2+2)+k3+3) from d_table group by k1+1 order by k1+1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1+1,sum(abs(k2+2)+k3+3) from d_table group by k1+1 order by k1+1;", "k1p2ap3ps") } diff --git a/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy b/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy index 016dc03753bd9e..69f80e4c96af9e 100644 --- a/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy +++ b/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy @@ -36,28 +36,25 @@ suite ("mv_with_view") { sql """insert into d_table select 1,1,1,'a';""" sql """insert into d_table select 2,2,2,'b';""" - createMV("create materialized view k132 as select k1,k3,k2 from d_table;") + createMV("create materialized view k312 as select k3,k1,k2 from d_table;") sql """insert into d_table select 3,-3,null,'c';""" - explain { - sql("select * from d_table order by k1;") - contains "(d_table)" - } + sql "analyze table d_table with sync;" + sql """set enable_stats=false;""" + + mv_rewrite_fail("select * from d_table order by k1;", "k312") qt_select_star "select * from d_table order by k1;" sql """ - drop view if exists v_k132; + drop view if exists v_k312; """ sql """ - create view v_k132 as select k1,k3,k2 from d_table where k1 = 1; + create view v_k312 as select k1,k3,k2 from d_table where k3 = 1; """ - explain { - sql("select * from v_k132 order by k1;") - contains "(k132)" - } - qt_select_mv "select * from v_k132 order by k1;" + mv_rewrite_success("select * from v_k312 order by k1;", "k312") + qt_select_mv "select * from v_k312 order by k1;" sql """ drop view if exists v_k124; @@ -66,9 +63,6 @@ suite ("mv_with_view") { sql """ create view v_k124 as select k1,k2,k4 from d_table where k1 = 1; """ - explain { - sql("select * from v_k124 order by k1;") - contains "(d_table)" - } + mv_rewrite_fail("select * from v_k124 order by k1;", "k312") qt_select_mv "select * from v_k124 order by k1;" } diff --git a/regression-test/suites/mv_p0/null_insert/null_insert.groovy b/regression-test/suites/mv_p0/null_insert/null_insert.groovy index 015406d80a3796..7b6ea058f2d90b 100644 --- a/regression-test/suites/mv_p0/null_insert/null_insert.groovy +++ b/regression-test/suites/mv_p0/null_insert/null_insert.groovy @@ -76,6 +76,7 @@ suite ("null_insert") { GROUP BY date,vid,os,ver,ip_country;""" sql """set enable_stats=true;""" + sql """alter table test modify column date set stats ('row_count'='3');""" mv_rewrite_success("""SELECT date, vid, os, ver, ip_country, hll_union(hll_hash(uid)) FROM test GROUP BY date,vid,os,ver,ip_country;""", "mv_test") diff --git a/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy b/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy index a872e23efc4cce..d34024ebdf5068 100644 --- a/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy +++ b/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy @@ -58,5 +58,6 @@ suite ("routine_load_hll") { qt_select_mv "select time_stamp, hll_union_agg(device_id) from test group by time_stamp order by 1;" sql """set enable_stats=true;""" + sql """alter table test modify column event_id set stats ('row_count'='2');""" mv_rewrite_success("select time_stamp, hll_union_agg(device_id) from test group by time_stamp order by 1;", "m_view") } diff --git a/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy b/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy index db95a084a69778..a83c78c25225c7 100644 --- a/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy +++ b/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy @@ -65,14 +65,6 @@ suite ("multiple_ssb_between") { ) ENGINE=OLAP DUPLICATE KEY(`LO_ORDERDATE`, `LO_ORDERKEY`) COMMENT "OLAP" - PARTITION BY RANGE(`LO_ORDERDATE`) - (PARTITION p1992 VALUES [("-2147483648"), ("19930101")), - PARTITION p1993 VALUES [("19930101"), ("19940101")), - PARTITION p1994 VALUES [("19940101"), ("19950101")), - PARTITION p1995 VALUES [("19950101"), ("19960101")), - PARTITION p1996 VALUES [("19960101"), ("19970101")), - PARTITION p1997 VALUES [("19970101"), ("19980101")), - PARTITION p1998 VALUES [("19980101"), ("19990101"))) DISTRIBUTED BY HASH(`LO_ORDERKEY`) BUCKETS 48 PROPERTIES ( "replication_num" = "1", @@ -153,17 +145,22 @@ suite ("multiple_ssb_between") { sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (1 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');""" qt_select_star "select * from lineorder_flat order by 1,2, P_MFGR;" + + sql "analyze table lineorder_flat with sync;" + sql """set enable_stats=true;""" + + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');""" - explain { - sql("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue + mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE LO_ORDERDATE >= 19930101 AND LO_ORDERDATE <= 19931231 AND LO_DISCOUNT BETWEEN 1 AND 3 - AND LO_QUANTITY < 25;""") - contains "(lineorder_q_1_1)" - } + AND LO_QUANTITY < 25;""", + "lineorder_q_1_1" + ) + qt_select_q_1_1 """SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE @@ -172,16 +169,16 @@ suite ("multiple_ssb_between") { AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25;""" - explain { - sql("""SELECT + mv_rewrite_success("""SELECT SUM(LO_REVENUE), (LO_ORDERDATE DIV 10000) AS YEAR, P_BRAND FROM lineorder_flat WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' GROUP BY (LO_ORDERDATE DIV 10000), P_BRAND - ORDER BY YEAR, P_BRAND;""") - contains "(lineorder_q_2_1)" - } + ORDER BY YEAR, P_BRAND;""", + "lineorder_q_2_1" + ) + qt_select_q_2_1 """SELECT SUM(LO_REVENUE), (LO_ORDERDATE DIV 10000) AS YEAR, P_BRAND @@ -190,8 +187,7 @@ suite ("multiple_ssb_between") { GROUP BY YEAR, P_BRAND ORDER BY YEAR, P_BRAND;""" - explain { - sql("""SELECT + mv_rewrite_success("""SELECT C_NATION, S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR, SUM(LO_REVENUE) AS revenue @@ -202,9 +198,9 @@ suite ("multiple_ssb_between") { AND LO_ORDERDATE >= 19920101 AND LO_ORDERDATE <= 19971231 GROUP BY C_NATION, S_NATION, YEAR - ORDER BY YEAR ASC, revenue DESC;""") - contains "(lineorder_q_3_1)" - } + ORDER BY YEAR ASC, revenue DESC;""", + "lineorder_q_3_1") + qt_select_q_3_1 """SELECT C_NATION, S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR, @@ -218,8 +214,7 @@ suite ("multiple_ssb_between") { GROUP BY C_NATION, S_NATION, YEAR ORDER BY YEAR ASC, revenue DESC;""" - explain { - sql("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR, + mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat @@ -228,9 +223,9 @@ suite ("multiple_ssb_between") { AND S_REGION = 'AMERICA' AND P_MFGR IN ('MFGR#1', 'MFGR#2') GROUP BY YEAR, C_NATION - ORDER BY YEAR ASC, C_NATION ASC;""") - contains "(lineorder_q_4_1)" - } + ORDER BY YEAR ASC, C_NATION ASC;""", + "lineorder_q_4_1") + qt_select_q_4_1 """SELECT (LO_ORDERDATE DIV 10000) AS YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit diff --git a/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy b/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy index d3ad529439971b..c7aa6e67c6ee8e 100644 --- a/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy @@ -63,14 +63,6 @@ suite ("mv_ssb_q_1_1") { ) ENGINE=OLAP DUPLICATE KEY(`LO_ORDERDATE`, `LO_ORDERKEY`) COMMENT "OLAP" - PARTITION BY RANGE(`LO_ORDERDATE`) - (PARTITION p1992 VALUES [("-2147483648"), ("19930101")), - PARTITION p1993 VALUES [("19930101"), ("19940101")), - PARTITION p1994 VALUES [("19940101"), ("19950101")), - PARTITION p1995 VALUES [("19950101"), ("19960101")), - PARTITION p1996 VALUES [("19960101"), ("19970101")), - PARTITION p1997 VALUES [("19970101"), ("19980101")), - PARTITION p1998 VALUES [("19980101"), ("19990101"))) DISTRIBUTED BY HASH(`LO_ORDERKEY`) BUCKETS 48 PROPERTIES ( "replication_num" = "1", @@ -80,7 +72,15 @@ suite ("mv_ssb_q_1_1") { ); """ - sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');""" + // Add more data when insert into firstly to make sure use mv easier + sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) + VALUES (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');""" + + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='6');""" createMV ("""create materialized view lineorder_q_1_1 as SELECT LO_ORDERKEY, SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue @@ -90,23 +90,25 @@ suite ("mv_ssb_q_1_1") { AND LO_ORDERDATE <= 19931231 AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3 AND LO_QUANTITY < 25 - GROUP BY - LO_ORDERKEY;""") - - sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" + GROUP BY LO_ORDERKEY;""") + sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) + VALUES + (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'), + (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'), + (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'), + (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" qt_select_star "select * from lineorder_flat order by 1,2, P_MFGR;" - explain { - sql("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue + sql "analyze table lineorder_flat with sync;" + mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE LO_ORDERDATE >= 19930101 AND LO_ORDERDATE <= 19931231 AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3 - AND LO_QUANTITY < 25;""") - contains "(lineorder_q_1_1)" - } + AND LO_QUANTITY < 25;""", "lineorder_q_1_1") + qt_select_mv """SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE @@ -115,6 +117,7 @@ suite ("mv_ssb_q_1_1") { AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3 AND LO_QUANTITY < 25;""" + sql""" drop materialized view lineorder_q_1_1 on lineorder_flat; """ qt_select """SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue @@ -124,4 +127,5 @@ suite ("mv_ssb_q_1_1") { AND LO_ORDERDATE <= 19931231 AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3 AND LO_QUANTITY < 25;""" + } diff --git a/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy b/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy index 6ead8380eed59b..0ac5f0245ef8ae 100644 --- a/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy @@ -93,6 +93,8 @@ suite ("mv_ssb_q_2_1") { qt_select_star "select * from lineorder_flat order by 1,2,P_MFGR;" + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');""" + mv_rewrite_success("""SELECT SUM(LO_REVENUE), (LO_ORDERDATE DIV 10000) AS YEAR, P_BRAND diff --git a/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy b/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy index 32765932317701..6b333d97050630 100644 --- a/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy @@ -66,14 +66,6 @@ suite ("mv_ssb_q_3_1") { ) ENGINE=OLAP DUPLICATE KEY(`LO_ORDERDATE`, `LO_ORDERKEY`) COMMENT "OLAP" - PARTITION BY RANGE(`LO_ORDERDATE`) - (PARTITION p1992 VALUES [("-2147483648"), ("19930101")), - PARTITION p1993 VALUES [("19930101"), ("19940101")), - PARTITION p1994 VALUES [("19940101"), ("19950101")), - PARTITION p1995 VALUES [("19950101"), ("19960101")), - PARTITION p1996 VALUES [("19960101"), ("19970101")), - PARTITION p1997 VALUES [("19970101"), ("19980101")), - PARTITION p1998 VALUES [("19980101"), ("19990101"))) DISTRIBUTED BY HASH(`LO_ORDERKEY`) BUCKETS 48 PROPERTIES ( "replication_num" = "1", @@ -83,7 +75,8 @@ suite ("mv_ssb_q_3_1") { ); """ - sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19920101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' ,'ASIA', 'ASIA', 'MFGR#12', 'MFGR#12', 'brand', 'color', 'type', 4 ,'container');""" + sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) + VALUES (19920101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' ,'ASIA', 'ASIA', 'MFGR#12', 'MFGR#12', 'brand', 'color', 'type', 4 ,'container');""" createMV ("""create materialized view lineorder_q_3_1 as SELECT @@ -98,12 +91,16 @@ suite ("mv_ssb_q_3_1") { AND LO_ORDERDATE <= 19971231 GROUP BY C_NATION, S_NATION, YEAR;""") - sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" + sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) + VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" qt_select_star "select * from lineorder_flat order by 1,2,P_MFGR;" - explain { - sql("""SELECT + sql """analyze table lineorder_flat with sync;""" + + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');""" + + mv_rewrite_success("""SELECT C_NATION, S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR, SUM(LO_REVENUE) AS revenue @@ -114,9 +111,9 @@ suite ("mv_ssb_q_3_1") { AND LO_ORDERDATE >= 19920101 AND LO_ORDERDATE <= 19971231 GROUP BY C_NATION, S_NATION, YEAR - ORDER BY YEAR ASC, revenue DESC;""") - contains "(lineorder_q_3_1)" - } + ORDER BY YEAR ASC, revenue DESC;""", + "lineorder_q_3_1") + qt_select_mv """SELECT C_NATION, S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR, diff --git a/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy b/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy index c094e8e00c9eaf..361f60997e3da6 100644 --- a/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy @@ -92,6 +92,8 @@ suite ("mv_ssb_q_4_1") { sql """analyze table lineorder_flat with sync;""" + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');""" + mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit diff --git a/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy b/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy index 1baf21b547f062..ebeb23d96e75e5 100644 --- a/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy @@ -119,6 +119,7 @@ suite ("q_4_1_r1") { GROUP BY YEAR, C_NATION ORDER BY YEAR ASC, C_NATION ASC;""" sql """set enable_stats=true;""" + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');""" mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit diff --git a/regression-test/suites/mv_p0/sum_count/sum_count.groovy b/regression-test/suites/mv_p0/sum_count/sum_count.groovy index b47e51999b7c42..137aabc4e16a89 100644 --- a/regression-test/suites/mv_p0/sum_count/sum_count.groovy +++ b/regression-test/suites/mv_p0/sum_count/sum_count.groovy @@ -51,6 +51,7 @@ suite ("sum_count") { sql "analyze table d_table with sync;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='8');""" mv_rewrite_success("select k1,k4,sum(k2),count(k2) from d_table group by k1,k4;", "kavg") qt_select_mv "select k1,k4,sum(k2),count(k2) from d_table group by k1,k4 order by 1,2;" diff --git a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy index 4e6374e300b09e..85a3f7137b9e40 100644 --- a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy +++ b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy @@ -46,6 +46,7 @@ suite ("sum_devide_count") { sql """analyze table d_table with sync;""" sql """set enable_stats=false;""" + mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;", "kavg") qt_select_mv "select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;" @@ -59,6 +60,7 @@ suite ("sum_devide_count") { qt_select_mv "select sum(k2)/count(k2) from d_table;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;", "kavg") mv_rewrite_success("select k1,sum(k2)/count(k2) from d_table group by k1 order by k1;", "kavg") diff --git a/regression-test/suites/mv_p0/test_28741/test_28741.groovy b/regression-test/suites/mv_p0/test_28741/test_28741.groovy index 9fabca41b943fc..88fb3c057da092 100644 --- a/regression-test/suites/mv_p0/test_28741/test_28741.groovy +++ b/regression-test/suites/mv_p0/test_28741/test_28741.groovy @@ -69,8 +69,8 @@ suite ("test_28741") { sql """set enable_stats=false;""" mv_rewrite_fail("select b1 from test where t >= '2023-12-20 17:21:00'", "mv_test") - qt_select "select b1 from test where t >= '2023-12-20 17:21:00'" sql """set enable_stats=true;""" + sql """alter table test modify column a set stats ('row_count'='2');""" mv_rewrite_fail("select b1 from test where t >= '2023-12-20 17:21:00'", "mv_test") } diff --git a/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy b/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy index 752cf40d9e5f6a..68952da602aa81 100644 --- a/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy +++ b/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy @@ -50,6 +50,7 @@ suite ("test_approx_count_distinct") { qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/test_base/test_base.groovy b/regression-test/suites/mv_p0/test_base/test_base.groovy index d2c018febd5ada..97ece8cd6cca68 100644 --- a/regression-test/suites/mv_p0/test_base/test_base.groovy +++ b/regression-test/suites/mv_p0/test_base/test_base.groovy @@ -53,6 +53,7 @@ suite ("test_base") { qt_select_mv "SELECT id,created_at FROM dwd order by 1, 2;" sql """set enable_stats=true;""" + sql """alter table dwd modify column id set stats ('row_count'='2');""" mv_rewrite_success("SELECT created_at, id FROM dwd where created_at = '2020-09-09 00:00:00' order by 1, 2;", "dwd_mv") mv_rewrite_success("SELECT id,created_at FROM dwd where id is not null order by 1, 2;", "dwd_mv") diff --git a/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy b/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy index eac65eeef22adf..0c9e0026cb7fbd 100644 --- a/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy +++ b/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy @@ -40,5 +40,6 @@ suite ("test_casewhen") { qt_select_mv "select store_id, sum(case when sale_amt>10 then 1 else 2 end) from sales_records group by store_id order by 1;" sql """set enable_stats=true;""" + sql """alter table sales_records modify column record_id set stats ('row_count'='4');""" mv_rewrite_success("select store_id, sum(case when sale_amt>10 then 1 else 2 end) from sales_records group by store_id order by 1;", "store_amt") } diff --git a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy index 776bf2ecf0bd17..8ee24cb5ba2bd9 100644 --- a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy +++ b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy @@ -49,6 +49,8 @@ suite("test_create_mv") { sql """ insert into ${tableName} values ('2024-03-20 10:00:00', 'a', 'b', 1) """ + sql """alter table test_mv_10010 modify column load_time set stats ('row_count'='1');""" + sql """ create materialized view mv_1 as select diff --git a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy index 12790500d2f98e..1e07d69ba82871 100644 --- a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy +++ b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy @@ -37,6 +37,8 @@ suite ("create_mv_complex_type") { sql """insert into base_table select 1, 100000, 1.0, '{"jsonk1": 123}', [100, 200], {"k1": 10}, {1, 2};""" + sql """alter table base_table modify column c_int set stats ('row_count'='1');""" + def success = false // 1. special column - mv dup key diff --git a/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy b/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy index 5e99464018e8d7..517b4415bd69ea 100644 --- a/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy +++ b/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy @@ -57,6 +57,7 @@ suite ("test_doc_e4") { qt_select_mv "select year(k4)+month(k4) from d_table where year(k4) = 2020 order by 1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='3');""" mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by 1,2;", "k1a2p2ap3ps") mv_rewrite_success("select bin(abs(k1)+k2+1),sum(abs(k2+2)+k3+3) from d_table group by bin(abs(k1)+k2+1);", "k1a2p2ap3ps") diff --git a/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy b/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy index 5e917e9b0f5bd0..23aeeb207f6898 100644 --- a/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy +++ b/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy @@ -53,6 +53,7 @@ suite ("test_dup_group_by_mv_abs") { qt_select_mv_sub "select sum(abs(k2)) from d_table group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,sum(abs(k2)) from d_table group by k1;", "k12sa") mv_rewrite_success("select sum(abs(k2)) from d_table group by k1;", "k12sa") } diff --git a/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy b/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy index ba9f60f6df5026..3551bb8a7fb359 100644 --- a/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy +++ b/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy @@ -53,6 +53,7 @@ suite ("test_dup_group_by_mv_plus") { qt_select_mv_sub "select sum(k2+1) from d_table group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,sum(k2+1) from d_table group by k1;", "k12sp") mv_rewrite_success("select sum(k2+1) from d_table group by k1;", "k12sp") diff --git a/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy b/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy index 89088ffb9ea317..f44c9d134f91f0 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy @@ -65,6 +65,7 @@ suite ("test_dup_mv_abs") { qt_select_group_mv_not "select sum(abs(k2)) from d_table group by k3 order by k3;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,abs(k2) from d_table order by k1;", "k12a") mv_rewrite_success("select abs(k2) from d_table order by k1;", "k12a") diff --git a/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy b/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy index a7db2617fe6067..4b8fa5aa6359e8 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy @@ -64,6 +64,7 @@ suite ("test_dup_mv_bin") { qt_select_group_mv_not "select group_concat(bin(k2)) from d_table group by k3 order by k3;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,bin(k2) from d_table order by k1;", "k12b") mv_rewrite_success("select bin(k2) from d_table order by k1;", "k12b") diff --git a/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy b/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy index 659f44de1e93c4..6530fd84cac7a8 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy @@ -46,6 +46,7 @@ suite ("test_dup_mv_bitmap_hash") { mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;", "k1g2bm") qt_select_mv "select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;", "k1g2bm") createMV "create materialized view k1g3bm as select k1,bitmap_union(bitmap_hash(k3)) from d_table group by k1;" @@ -58,11 +59,13 @@ suite ("test_dup_mv_bitmap_hash") { qt_select_star "select * from d_table order by k1,k2,k3;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" sql """analyze table d_table with sync;""" sql """set enable_stats=false;""" mv_rewrite_success("select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1;", "k1g3bm") qt_select_mv_sub "select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1;", "k1g3bm") } diff --git a/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy b/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy index 5abe03e736b641..ad437db464b3bb 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy @@ -70,7 +70,7 @@ suite ("test_dup_mv_plus") { qt_select_mv "select k1,k2+1 from d_table order by k2;" sql """set enable_stats=true;""" - + sql """alter table d_table modify column k4 set stats ('row_count'='3');""" mv_rewrite_success("select k1,k2+1 from d_table order by k1;", "k12p") mv_rewrite_success("select k2+1 from d_table order by k1;", "k12p") diff --git a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy index 562fba25b2e8a9..5ccdae1bd8a7c5 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy @@ -47,6 +47,7 @@ suite ("test_dup_mv_repeat") { qt_select_mv "SELECT s AS s, sum(n) / count(DISTINCT dt) AS n FROM db1 GROUP BY GROUPING SETS((s)) order by 1;" sql """set enable_stats=true;""" + sql """alter table db1 modify column dt set stats ('row_count'='2');""" mv_rewrite_success("SELECT s AS s, sum(n) / count(DISTINCT dt) AS n FROM db1 GROUP BY GROUPING SETS((s)) order by 1;", "dbviwe") } diff --git a/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy b/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy index 92a8d6520f3122..18f798f8b6aac3 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy @@ -45,6 +45,7 @@ suite ("test_dup_mv_year") { qt_select_mv "select k1,year(k2) from d_table order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,year(k2) from d_table order by k1;", "k12y") createMV "create materialized view k13y as select k1,year(k3) from d_table;" diff --git a/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy b/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy index 50f1fd61806f2e..2fadc0b0da91a0 100644 --- a/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy +++ b/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy @@ -56,6 +56,7 @@ suite ("test_mv_dp") { time 10000 // limit inflight 10s } */ + mv_rewrite_success("""select d, bitmap_union_count(bitmap_from_array(cast(uid_list as array))), bitmap_union_count(bitmap_from_array(if(status='success', cast(uid_list as array), array()))) @@ -68,6 +69,7 @@ suite ("test_mv_dp") { from dp group by d order by 1;""" sql """set enable_stats=true;""" + sql """alter table dp modify column d set stats ('row_count'='4');""" mv_rewrite_success("""select d, bitmap_union_count(bitmap_from_array(cast(uid_list as array))), bitmap_union_count(bitmap_from_array(if(status='success', cast(uid_list as array), array()))) diff --git a/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy b/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy index b9c79f8bc11635..35d082fbff6082 100644 --- a/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy +++ b/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy @@ -39,6 +39,8 @@ suite ("test_mv_mor") { sql "insert into u_table select 1,1,1,2;" sql "insert into u_table select 1,2,1,2;" + sql """alter table u_table modify column k1 set stats ('row_count'='2');""" + // do not match mv coz preagg is off, mv need contains all key column to make row count correct mv_rewrite_fail("select k1,k2+k3 from u_table order by k1;", "k123p") qt_select_mv "select k1,k2+k3 from u_table order by k1;" diff --git a/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy b/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy index d2c9921edefe7a..2d43785ba6fafb 100644 --- a/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy +++ b/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy @@ -50,6 +50,7 @@ suite ("test_ndv") { qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy b/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy index c6b1dd2d8a9b1c..12f5765666ef28 100644 --- a/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy +++ b/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy @@ -51,6 +51,7 @@ suite ("test_nvl") { qt_select_mv "select ifnull(id,0) from dwd order by 1;" sql """set enable_stats=true;""" + sql """alter table dwd modify column id set stats ('row_count'='2');""" mv_rewrite_success("select nvl(id,0) from dwd order by 1;", "dwd_mv") mv_rewrite_success("select ifnull(id,0) from dwd order by 1;", "dwd_mv") diff --git a/regression-test/suites/mv_p0/test_o2/test_o2.groovy b/regression-test/suites/mv_p0/test_o2/test_o2.groovy index 665156056c0096..34d31a08cfae7b 100644 --- a/regression-test/suites/mv_p0/test_o2/test_o2.groovy +++ b/regression-test/suites/mv_p0/test_o2/test_o2.groovy @@ -60,6 +60,7 @@ suite ("test_o2") { qt_select_mv "select ts,metric_name,platform,sum(count_value) from o2_order_events group by ts,metric_name,platform;" sql """set enable_stats=true;""" + sql """alter table o2_order_events modify column ts set stats ('row_count'='2');""" mv_rewrite_success("select ts,metric_name,platform,sum(count_value) from o2_order_events group by ts,metric_name,platform;", "o2_order_events_mv") } diff --git a/regression-test/suites/mv_p0/test_substr/test_substr.groovy b/regression-test/suites/mv_p0/test_substr/test_substr.groovy index c43c3ae6483acd..7d6ae82634fd8c 100644 --- a/regression-test/suites/mv_p0/test_substr/test_substr.groovy +++ b/regression-test/suites/mv_p0/test_substr/test_substr.groovy @@ -56,6 +56,7 @@ suite ("test_substr") { qt_select_mv "SELECT substr(created_at,1,10) as statistic_date, max(dt) as dt FROM dwd group by substr(created_at,1,10);" sql """set enable_stats=true;""" + sql """alter table dwd modify column id set stats ('row_count'='2');""" mv_rewrite_success("SELECT substr(created_at,1,10) as statistic_date, max(dt) as dt FROM dwd group by substr(created_at,1,10);", "dwd_mv") } diff --git a/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy b/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy index f65e36484e7de9..bbfa8ea6a54d17 100644 --- a/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy +++ b/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy @@ -66,6 +66,7 @@ suite ("test_tbl_name") { group by id order by 1,2; """ sql """set enable_stats=true;""" + sql """alter table functionality_olap modify column id set stats ('row_count'='2');""" mv_rewrite_success("""select functionality_olap.id as id, sum(functionality_olap.score) as score_max diff --git a/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy b/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy index b62ec696224197..69264d838e4d24 100644 --- a/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy +++ b/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy @@ -62,6 +62,7 @@ suite ("test_upper_alias") { qt_select_mv "SELECT d_a AS d_b FROM test_0401 order by 1;" sql """set enable_stats=true;""" + sql """alter table test_0401 modify column d_b set stats ('row_count'='3');""" mv_rewrite_any_success("SELECT upper(d_b) AS d_b FROM test_0401 GROUP BY upper(d_b) order by 1;", ["test_0401_mv", "test_0401_mv2"]) diff --git a/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy b/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy index 3d139cbc01bbe7..f971673971a2aa 100644 --- a/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy +++ b/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy @@ -53,6 +53,7 @@ suite ("test_user_activity") { qt_select_group_mv "select n_dx, percentile_approx(n_duration, 0.5) as p50, percentile_approx(n_duration, 0.90) as p90 FROM u_axx GROUP BY n_dx;" sql """set enable_stats=true;""" + sql """alter table u_axx modify column r_xx set stats ('row_count'='3');""" mv_rewrite_success("select n_dx, percentile_approx(n_duration, 0.5) as p50, percentile_approx(n_duration, 0.90) as p90 FROM u_axx GROUP BY n_dx;", "session_distribution_2") } diff --git a/regression-test/suites/mv_p0/unique/unique.groovy b/regression-test/suites/mv_p0/unique/unique.groovy index 1209d587c78941..d54329a5c54e61 100644 --- a/regression-test/suites/mv_p0/unique/unique.groovy +++ b/regression-test/suites/mv_p0/unique/unique.groovy @@ -73,6 +73,7 @@ suite ("unique") { qt_select_star "select * from u_table order by k1;" sql """set enable_stats=true;""" + sql """alter table u_table modify column k1 set stats ('row_count'='3');""" mv_rewrite_success("select k3,length(k1),k2 from u_table order by 1,2,3;", "k31l42") // todo: support match query diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy index 70b016de2da80f..f39e8df6cabe6a 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy @@ -36,6 +36,7 @@ suite ("testAggQueryOnAggMV1") { sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" +sql """alter table emps modify column time_col set stats ('row_count'='4');""" createMV("create materialized view emps_mv as select deptno, sum(salary), max(commission) from emps group by deptno;") createMV("create materialized view emps_mv_count_key as select deptno, count(deptno) from emps group by deptno;") diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy index a25ac24404e181..739dde33372cb6 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy @@ -51,6 +51,7 @@ suite ("testAggQueryOnAggMV10") { qt_select_mv "select deptno, commission, sum(salary) + 1 from emps group by rollup (deptno, commission) order by 1,2;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select deptno, commission, sum(salary) + 1 from emps group by rollup (deptno, commission);", diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy index c26db8e034e66a..a4d69136bc03ca 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy @@ -50,6 +50,7 @@ suite ("testAggQueryOnAggMV11") { qt_select_mv "select deptno, count(salary) + count(1) from emps group by deptno order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_fail("select deptno, count(salary) + count(1) from emps group by deptno;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy index 945e4e9194910f..5cf7d257fe8b73 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy @@ -53,6 +53,7 @@ suite ("testAggQueryOnAggMV2") { qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;", diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy index 46457f0d0a81d4..d2f61e04762b2f 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy @@ -38,7 +38,6 @@ suite ("testAggQueryOnAggMV3") { sql """insert into emps values("2020-01-04",4,"d",21,4,4);""" - createMV("create materialized view emps_mv as select deptno, commission, sum(salary) from emps group by deptno, commission;") sql "analyze table emps with sync;" @@ -56,6 +55,7 @@ suite ("testAggQueryOnAggMV3") { qt_select_mv "select commission, sum(salary) from emps where commission = 100 group by commission order by commission;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select commission, sum(salary) from emps where deptno > 0 and commission * (deptno + commission) = 100 group by commission order by commission;", diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy index 1fa4342952cad4..1f14728e3b7b2c 100644 --- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy @@ -51,6 +51,7 @@ suite ("testAggQuqeryOnAggMV5") { qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where sum_salary>10 order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where sum_salary>0;", diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy index 3cca0273051f0a..4cafeaa185f4a3 100644 --- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy @@ -51,6 +51,7 @@ suite ("testAggQuqeryOnAggMV6") { qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps where deptno>=20 group by deptno) a where sum_salary>10 order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps where deptno>=0 group by deptno) a where sum_salary>10;", diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy index 1a606141e38f04..d43b5796e6d7f9 100644 --- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy @@ -50,6 +50,8 @@ suite ("testAggQuqeryOnAggMV7") { qt_select_mv "select deptno, sum(salary) from emps where deptno>=20 group by deptno order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select deptno, sum(salary) from emps where deptno>=20 group by deptno;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy b/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy index 7f2427e275de47..6dc78784956d32 100644 --- a/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy @@ -50,6 +50,7 @@ suite ("testAggregateMVCalcAggFunctionQuery") { qt_select_mv "select deptno, sum(salary + 1) from emps where deptno > 10 group by deptno order by deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_fail("select deptno, sum(salary + 1) from emps where deptno > 10 group by deptno;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy index 69410ae7be5d0d..c1287bfa94bcc4 100644 --- a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy @@ -51,7 +51,7 @@ suite ("testBitmapUnionInQuery") { qt_select_mv "select user_id, bitmap_count(bitmap_union(to_bitmap(tag_id))) a from user_tags group by user_id having a>1 order by a;" sql """set enable_stats=true;""" - + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_success("select user_id, bitmap_union_count(to_bitmap(tag_id)) a from user_tags group by user_id having a>1 order by a;", diff --git a/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy b/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy index 12152aad369e1d..5cc05db62be1a7 100644 --- a/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy +++ b/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy @@ -66,6 +66,10 @@ suite ("testCountDistinctToBitmap") { sql """insert into user_tags2 values("2020-01-01",1,"a",1);""" sql """insert into user_tags2 values("2020-01-02",2,"b",2);""" + + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """alter table user_tags2 modify column time_col set stats ('row_count'='3');""" + createMV("create materialized view user_tags_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags2 group by user_id;") sql """insert into user_tags2 values("2020-01-01",1,"a",2);""" diff --git a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy index c671e106a10648..698490f0943e88 100644 --- a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy +++ b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy @@ -47,6 +47,7 @@ suite ("testIncorrectMVRewriteInSubquery") { qt_select_mv "select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags where user_name in (select user_name from user_tags group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_fail("select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags where user_name in (select user_name from user_tags group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;", diff --git a/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy b/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy index dbece0c9592364..45230c5b5587d8 100644 --- a/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy +++ b/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy @@ -45,6 +45,7 @@ suite ("testIncorrectRewriteCountDistinct") { qt_select_mv "select user_name, count(distinct tag_id) from user_tags group by user_name order by user_name;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_fail("select user_name, count(distinct tag_id) from user_tags group by user_name;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy b/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy index 3caf0000c1824d..8f41fc1ffb5e74 100644 --- a/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy +++ b/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy @@ -43,6 +43,8 @@ suite ("testJoinOnLeftProjectToJoin") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """alter table depts modify column time_col set stats ('row_count'='3');""" + sql """insert into depts values("2020-01-02",2,"b",2);""" sql """insert into depts values("2020-01-03",3,"c",3);""" sql """insert into depts values("2020-01-02",2,"b",1);""" @@ -59,6 +61,7 @@ suite ("testJoinOnLeftProjectToJoin") { qt_select_mv "select * from (select deptno , sum(salary) from emps group by deptno) A join (select deptno, max(cost) from depts group by deptno ) B on A.deptno = B.deptno order by A.deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_all_success("select * from (select deptno , sum(salary) from emps group by deptno) A join (select deptno, max(cost) from depts group by deptno ) B on A.deptno = B.deptno;", ["emps_mv", "depts_mv"]) } diff --git a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy index a9456cb4e79c96..c77f58609ca0cb 100644 --- a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy +++ b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy @@ -50,6 +50,7 @@ suite ("testNDVToHll") { qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy b/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy index 2aa88d84f3d433..41cf480e880cd0 100644 --- a/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy +++ b/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy @@ -49,6 +49,7 @@ suite ("testOrderByQueryOnProjectView") { qt_select_mv "select empid from emps order by deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select empid from emps where deptno > 0 order by deptno;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy index d95c3e2fc6de88..d578cbddba945d 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy @@ -61,6 +61,7 @@ suite ("testProjectionMV1") { qt_select_mv "select deptno, sum(empid) from emps group by deptno order by deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select empid, deptno from emps where deptno > 0 order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy index 2f7548b14c8635..b663bfcd2b1738 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy @@ -51,6 +51,7 @@ suite ("testProjectionMV2") { qt_select_base "select name from emps where deptno -1 = 0 order by empid;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select empid + 1 from emps where deptno = 1 order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy index e4ea1aa4de023e..378ec80d6cfd72 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy @@ -53,6 +53,7 @@ suite ("testProjectionMV3") { qt_select_mv2 "select name from emps where deptno = 1 order by empid;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select empid + 1, name from emps where deptno = 1 order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy index 0c6f737deabb29..37a57f80d03cae 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy @@ -52,6 +52,7 @@ suite ("testProjectionMV4") { qt_select_base "select empid from emps where deptno > 1 and empid > 1 order by empid;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_fail("select empid from emps where deptno > 1 and empid > 1 order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy b/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy index 57ddba4d3c8f72..15394870f9b11c 100644 --- a/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy +++ b/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy @@ -70,4 +70,5 @@ suite ("testQueryOnStar") { contains "(emps_mv)" } qt_select_mv "select ref_1.`empid` as c0 from tpch_tiny_region as ref_0 left join emps as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;" + } diff --git a/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy b/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy index 0cfd80fcb06b4e..9e3bd886b31ea5 100644 --- a/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy +++ b/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy @@ -29,6 +29,8 @@ suite ("testSelectMVWithTableAlias") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """insert into user_tags values("2020-01-01",1,"a",1);""" sql """insert into user_tags values("2020-01-02",2,"b",2);""" diff --git a/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy b/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy index 006b72885ec112..73ddb21109bb91 100644 --- a/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy +++ b/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy @@ -52,6 +52,7 @@ suite ("testSingleMVMultiUsage") { } qt_select_mv "select * from (select deptno, empid from emps where deptno>100) A join (select deptno, empid from emps where deptno >200) B using (deptno) order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") explain { diff --git a/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy b/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy index 6c1747059d1ae3..cd134a5d48ea40 100644 --- a/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy @@ -38,7 +38,6 @@ suite ("testSubQuery") { sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" - createMV("create materialized view emps_mv as select deptno, empid from emps;") sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" @@ -52,5 +51,6 @@ suite ("testSubQuery") { qt_select_mv "select empid, deptno, salary from emps e1 where empid = (select max(empid) from emps where deptno = e1.deptno) order by deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") } diff --git a/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy b/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy index f504c81f214e91..c6243a3f923616 100644 --- a/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy +++ b/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy @@ -35,6 +35,7 @@ suite ("testUnionDistinct") { sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + createMV("create materialized view emps_mv as select empid, deptno from emps order by empid, deptno;") sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" @@ -52,6 +53,7 @@ suite ("testUnionDistinct") { } qt_select_mv "select * from (select empid, deptno from emps where empid >1 union select empid, deptno from emps where empid <0) t order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") explain { diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy index c51b7cc95d201b..f643f551af83d5 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy @@ -57,6 +57,8 @@ suite("agg_on_none_agg") { (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); """ + sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');""" + sql """ drop table if exists lineitem """ @@ -95,6 +97,8 @@ suite("agg_on_none_agg") { (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); """ + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """ drop table if exists partsupp """ @@ -120,6 +124,8 @@ suite("agg_on_none_agg") { (2, 3, 10, 11.01, 'supply2'); """ + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + sql """analyze table orders with sync;""" sql """analyze table lineitem with sync;""" sql """analyze table partsupp with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy index 050662e19190c4..27a8b693dfe9ac 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy @@ -95,6 +95,8 @@ suite("agg_variety") { (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); """ + + sql """ drop table if exists partsupp """ @@ -124,6 +126,10 @@ suite("agg_variety") { sql """analyze table lineitem with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // query dimension is less then mv def mv1_0 = """ select diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index cda9b21fc4e51d..5b8aa6e00d2308 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -132,6 +132,9 @@ suite("aggregate_with_roll_up") { sql """analyze table partsupp with sync""" sql """analyze table lineitem with sync""" sql """analyze table orders with sync""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" // multi table // filter inside + left + use roll up dimension diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 85f85101961a97..bed1f1dcaa39ce 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -45,11 +45,9 @@ suite("aggregate_without_roll_up") { "replication_num" = "1" ); """ - sql """ drop table if exists lineitem """ - sql""" CREATE TABLE IF NOT EXISTS lineitem ( l_orderkey INTEGER NOT NULL, @@ -76,11 +74,9 @@ suite("aggregate_without_roll_up") { "replication_num" = "1" ) """ - sql """ drop table if exists partsupp """ - sql """ CREATE TABLE IF NOT EXISTS partsupp ( ps_partkey INTEGER NOT NULL, @@ -124,6 +120,12 @@ suite("aggregate_without_roll_up") { (2, 3, 10, 11.01, 'supply2', null); """ + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // single table // with filter def mv1_0 = "select o_shippriority, o_comment, " + @@ -1507,4 +1509,463 @@ suite("aggregate_without_roll_up") { order_qt_query28_0_after "${query28_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv28_0""" + + + // query and mv has the same filter but position is different, should rewrite successfully + def mv29_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + lineitem.public_col is null + or lineitem.public_col <> 1 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + def query29_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + lineitem.public_col is null + or lineitem.public_col <> 1 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + order_qt_query29_0_before "${query29_0}" + async_mv_rewrite_success(db, mv29_0, query29_0, "mv29_0") + order_qt_query29_0_after "${query29_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" + + + // query and mv has the same filter but position is different, should rewrite successfully + // mv join condition has alias + def mv30_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey_alias, + orders.public_col as col1, + l_orderkey_alias, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey_alias, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2, + sum( + o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey as o_orderkey_alias, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey as l_orderkey_alias, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on lineitem.l_orderkey_alias = orders.o_orderkey_alias + inner join ( + select + ps_partkey as ps_partkey_alias, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on partsupp.ps_partkey_alias = orders.o_orderkey_alias + where + lineitem.public_col is null + or lineitem.public_col <> 1 + and o_orderkey_alias = 2 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + def query30_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + lineitem.public_col is null + or lineitem.public_col <> 1 + and o_orderkey = 2 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + order_qt_query30_0_before "${query30_0}" + async_mv_rewrite_success(db, mv30_0, query30_0, "mv30_0") + order_qt_query30_0_after "${query30_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv30_0""" + + + // query and mv has the same filter but position is different, should rewrite successfully + // query join condition has alias + def mv31_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + lineitem.public_col is null + or lineitem.public_col <> 1 + and o_orderkey = 2 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + def query31_0 = """ +select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey_alias, + orders.public_col as col1, + l_orderkey_alias, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey_alias, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2, + sum( + o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey as o_orderkey_alias, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey as l_orderkey_alias, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on lineitem.l_orderkey_alias = orders.o_orderkey_alias + inner join ( + select + ps_partkey as ps_partkey_alias, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on partsupp.ps_partkey_alias = orders.o_orderkey_alias + where + lineitem.public_col is null + or lineitem.public_col <> 1 + and o_orderkey_alias = 2 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + order_qt_query31_0_before "${query31_0}" + async_mv_rewrite_success(db, mv31_0, query31_0, "mv31_0") + order_qt_query31_0_after "${query31_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv31_0""" } diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy index 5a6c75f6c59c6e..b9e671f2636b82 100644 --- a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy @@ -49,6 +49,7 @@ suite("grace_period") { "replication_num" = "1" ); """ + sql """alter table orders_partition modify column o_comment set stats ('row_count'='3');""" sql """ drop table if exists lineitem_partition @@ -81,6 +82,7 @@ suite("grace_period") { "replication_num" = "1" ); """ + sql """alter table lineitem_partition modify column l_comment set stats ('row_count'='3');""" sql """ insert into orders_partition values diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy index 72a8a691078a2d..4af1d778c002b2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy @@ -120,6 +120,15 @@ suite("materialized_view_switch") { (2, 3, 10, 11.01, 'supply2'); """ + sql """analyze table lineitem with sync; + analyze table orders with sync; + analyze table partsupp with sync; + """ + + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + def mv_name = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey from lineitem @@ -134,12 +143,17 @@ suite("materialized_view_switch") { where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5; """ - async_mv_rewrite_success(db, mv_name, query, "mv_name") + async_mv_rewrite_success(db, mv_name, query, "mv_name_1") + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name_1""" + sql "SET enable_materialized_view_rewrite=false" - mv_not_part_in(query, "mv_name") + create_async_mv(db, "mv_name_2", mv_name) + mv_not_part_in(query, "mv_name_2") + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name_2""" + sql "SET enable_materialized_view_rewrite=true" - async_mv_rewrite_success(db, mv_name, query, "mv_name") - sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name""" + async_mv_rewrite_success(db, mv_name, query, "mv_name_3") + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name_3""" // test when materialized_view_relation_mapping_max_count is 8 def mv1_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy index f2b5ad09c0b17a..3aed3b0f9e24df 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy @@ -98,6 +98,9 @@ suite("partition_mv_rewrite_dimension_1") { (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'); """ + sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');""" + sql """analyze table orders_1 with sync;""" sql """analyze table lineitem_1 with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy index 53f8777a1589d7..c7ee359cdef2e4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_3") { sql """analyze table orders_2_3 with sync;""" sql """analyze table lineitem_2_3 with sync;""" + sql """alter table orders_2_3 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_3 modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy index deb6255119a26e..e59b2771dd4e57 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy @@ -124,6 +124,10 @@ suite("partition_mv_rewrite_dimension_2_4") { (3, null, 1, 99.5, 'yy'); """ + sql """alter table orders_2_4 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_4 modify column l_comment set stats ('row_count'='7');""" + sql """alter table partsupp_2_4 modify column ps_comment set stats ('row_count'='3');""" + sql """analyze table orders_2_4 with sync;""" sql """analyze table lineitem_2_4 with sync;""" sql """analyze table partsupp_2_4 with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy index 5d8a6b5270285b..e5e43073bede3e 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy @@ -124,6 +124,10 @@ suite("partition_mv_rewrite_dimension_2_5") { (3, null, 1, 99.5, 'yy'); """ + sql """alter table orders_2_5 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_5 modify column l_comment set stats ('row_count'='7');""" + sql """alter table partsupp_2_5 modify column ps_comment set stats ('row_count'='3');""" + sql """analyze table orders_2_5 with sync;""" sql """analyze table lineitem_2_5 with sync;""" sql """analyze table partsupp_2_5 with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy index 1831e8de9a7a9d..f3289b007a0505 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy @@ -124,6 +124,12 @@ suite("partition_mv_rewrite_dimension_2_6") { (3, null, 1, 99.5, 'yy'); """ + sql """alter table orders_2_6 modify column o_comment set stats ('row_count'='10');""" + + sql """alter table lineitem_2_6 modify column l_comment set stats ('row_count'='7');""" + + sql """alter table partsupp_2_6 modify column ps_comment set stats ('row_count'='3');""" + sql """analyze table orders_2_6 with sync;""" sql """analyze table lineitem_2_6 with sync;""" sql """analyze table partsupp_2_6 with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy index 6def436e11c5c4..e36a5e277daca7 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy @@ -102,6 +102,8 @@ suite("partition_mv_rewrite_dimension_2_full_join") { sql """analyze table orders_2_full_join with sync;""" sql """analyze table lineitem_2_full_join with sync;""" + sql """alter table orders_2_full_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_full_join modify column l_comment set stats ('row_count'='7');""" def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy index 526e05f0692fbe..a615c7316bdb56 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_2") { sql """analyze table orders_2_2 with sync;""" sql """analyze table lineitem_2_2 with sync;""" + sql """alter table orders_2_2 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_2 modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy index 0db7b3bf008c68..35d48270574a6f 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_left_anti_join") { sql """analyze table orders_2_left_anti_join with sync;""" sql """analyze table lineitem_2_left_anti_join with sync;""" + sql """alter table orders_2_left_anti_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_left_anti_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy index f59dffd0091bb4..616707412954e4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_1") { sql """analyze table orders_2_1 with sync;""" sql """analyze table lineitem_2_1 with sync;""" + sql """alter table orders_2_1 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_1 modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy index 5e60250ef91d26..f203db1134f2f2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_left_semi_join") { sql """analyze table orders_2_left_semi_join with sync;""" sql """analyze table lineitem_2_left_semi_join with sync;""" + sql """alter table orders_2_left_semi_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_left_semi_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy index 289ed57287f1d8..c6097d845f41c9 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_anti_join") { sql """analyze table orders_2_right_anti_join with sync;""" sql """analyze table lineitem_2_right_anti_join with sync;""" + sql """alter table orders_2_right_anti_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_right_anti_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy index 8d6abb6dc561f8..850de42fc696eb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_join") { sql """analyze table orders_2_right_join with sync;""" sql """analyze table lineitem_2_right_join with sync;""" + sql """alter table orders_2_right_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_right_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy index 8f1f6f48a0ae91..97ddcd0eb1da29 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_semi_join") { sql """analyze table orders_2_right_semi_join with sync;""" sql """analyze table lineitem_2_right_semi_join with sync;""" + sql """alter table orders_2_right_semi_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_right_semi_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy index f433e18f48c062..776f1b996f1826 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy @@ -101,6 +101,9 @@ suite("partition_mv_rewrite_dimension_self_conn") { sql """analyze table orders_self_conn with sync;""" sql """analyze table lineitem_self_conn with sync;""" + sql """alter table orders_self_conn modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_self_conn modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy index dee43f20f4926a..fd41789d02661e 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy @@ -101,6 +101,9 @@ suite("dimension_2_join_agg_replenish") { sql """analyze table orders with sync;""" sql """analyze table lineitem with sync;""" + sql """alter table orders modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt @@ -649,19 +652,19 @@ suite("dimension_2_join_agg_replenish") { """ def right_semi_mv_stmt_8 = """select t1.l_orderkey, t1.l_shipdate, t1.l_partkey from orders - right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 on t1.l_orderkey = orders.o_orderkey group by t1.l_orderkey, t1.l_shipdate, t1.l_partkey """ def right_semi_mv_stmt_9 = """select t1.l_orderkey, t1.l_shipdate, t1.l_partkey from orders - right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 on t1.l_orderkey = orders.o_orderkey group by t1.l_orderkey, t1.l_shipdate, t1.l_partkey """ - def right_semi_mv_stmt_10 = """select t1.l_orderkey + def right_semi_mv_stmt_10 = """select t1.sum_total, max_total+min_total as col3, count_all from orders right semi join (select l_orderkey, sum(l_quantity) as sum_total, max(l_quantity) as max_total, @@ -670,10 +673,10 @@ suite("dimension_2_join_agg_replenish") { sum(l_quantity) + count(*) as col5, bitmap_union(to_bitmap(case when l_quantity > 1 and l_orderkey IN (1, 3) then l_partkey else null end)) as cnt_1, bitmap_union(to_bitmap(case when l_quantity > 2 and l_orderkey IN (2) then l_partkey else null end)) as cnt_2 - from lineitem where l_orderkey > 1 + 1 group by l_orderkey) as t1 + from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey) as t1 on t1.l_orderkey = orders.o_orderkey group by - t1.l_orderkey + t1.sum_total, col3, count_all """ def left_anti_mv_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey @@ -790,7 +793,7 @@ suite("dimension_2_join_agg_replenish") { bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 from orders where o_orderkey >= 1 + 1 group by o_orderkey) as t1 right anti join lineitem on lineitem.l_orderkey = t1.o_orderkey - group by l_orderkey, l_shipdate, l_partkey + group by l_orderkey, l_shipdate, l_partkey """ def right_anti_mv_stmt_6 = """select t1.l_shipdate, t1.l_quantity, t1.l_orderkey diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy index 04bd6ac5c6ac8d..3afbe42fd2a1c3 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy @@ -101,6 +101,9 @@ suite("dimension_join_agg_negative") { sql """analyze table orders_negative with sync;""" sql """analyze table lineitem_negative with sync;""" + sql """alter table orders_negative modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_negative modify column l_comment set stats ('row_count'='7');""" + // left join + agg (function + group by + +-*/ + filter) def left_mv_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority) as t1 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy index 763880ea94be44..2ca4a524989c99 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy @@ -99,6 +99,9 @@ suite("filter_equal_or_notequal_case") { sql """analyze table orders_1 with sync;""" sql """analyze table lineitem_1 with sync;""" + sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy index ccab3aaf35360c..80645f6cebf013 100644 --- a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy @@ -131,6 +131,9 @@ suite("direct_query_mv") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='15');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" create_async_mv(db, "mv1_0", """ diff --git a/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy b/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy index ec8cecbaf517d0..4655f9d004bc7a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy @@ -127,6 +127,9 @@ suite("materialized_view_grouping_sets") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table orders modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='7');""" + // query has group sets, and mv doesn't // single table grouping sets without grouping scalar function def mv1_0 = diff --git a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy new file mode 100644 index 00000000000000..15d93e32f65dc2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert + +suite("is_in_debug_mode") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders; + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + UNIQUE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + create_async_mv(db, "basic_mv", """ + select * from orders where o_orderkey > 1; + """) + + sql """set skip_delete_sign = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set skip_delete_sign = false;""" + + + sql """set skip_storage_engine_merge = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set skip_storage_engine_merge = false;""" + + + sql """set skip_delete_bitmap = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set skip_delete_bitmap = false;""" + + + sql """set skip_delete_predicate = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set skip_delete_predicate = false;""" + + + sql """set show_hidden_columns = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set show_hidden_columns = false;""" + + sql """drop materialized view if exists basic_mv""" +} + diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy index 2b018ebc37373b..8c0674faeb8c32 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy @@ -123,6 +123,10 @@ suite("inner_join_dphyp") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // without filter def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + "from lineitem " + diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy index 45d9aee067c798..71154fbeb276cf 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy @@ -123,6 +123,10 @@ suite("outer_join_dphyp") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // without filter def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + "from lineitem " + diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy index b745f7d0197c28..3aebf92b7cec71 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy @@ -122,6 +122,10 @@ suite("inner_join") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // without filter def mv1_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy index daebc78f23457c..7ec650c2ee2ddd 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy @@ -278,9 +278,20 @@ suite("outer_join") { sql """analyze table lineitem with sync;""" sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """analyze table lineitem_null with sync;""" + sql """analyze table orders_null with sync;""" sql """analyze table orders_same_col with sync;""" sql """analyze table lineitem_same_col with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table lineitem_same_col modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table orders_same_col modify column o_comment set stats ('row_count'='18');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + sql """alter table lineitem_null modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders_null modify column o_comment set stats ('row_count'='5');""" + + // without filter def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + "from lineitem " + diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy index 5688c2bbe21d6a..cce090b2a940fb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy @@ -106,6 +106,9 @@ suite("inner_join_infer_and_derive") { sql """analyze table orders_inner with sync;""" sql """analyze table lineitem_inner with sync;""" + sql """alter table orders_inner modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_inner modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy index dbfaa783383af8..7e51847c3b1a07 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy @@ -106,6 +106,8 @@ suite("left_join_infer_and_derive") { sql """analyze table orders_left with sync;""" sql """analyze table lineitem_left with sync;""" + sql """alter table orders_left modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_left modify column l_comment set stats ('row_count'='7');""" def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy index 4b5bdf467a5d02..1761efa5ef4475 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy @@ -106,6 +106,9 @@ suite("right_join_infer_and_derive") { sql """analyze table orders_right with sync;""" sql """analyze table lineitem_right with sync;""" + sql """alter table orders_right modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_right modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy b/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy new file mode 100644 index 00000000000000..9e80ea966fc1a2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy @@ -0,0 +1,204 @@ +package mv.micro_test +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("micro_test_when_cte") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate)( + FROM ('2023-12-01') TO ('2023-12-31') INTERVAL 1 DAY + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) + (FROM ('2023-12-01') TO ('2023-12-31') INTERVAL 1 DAY) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp with sync""" + sql """analyze table lineitem with sync""" + sql """analyze table orders with sync""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='18');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + + def query_sql = """ + WITH scan_data_cte as ( + select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey + from (select * from lineitem where L_LINENUMBER > 1) t1 + left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY + ) + SELECT * FROM scan_data_cte; + """ + def mv_sql = """ + WITH scan_data_cte as ( + select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey + from (select * from lineitem where L_LINENUMBER > 1) t1 + left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY + ) + SELECT * FROM scan_data_cte; + """ + def mv_name = """mv_with_cte_test""" + + // query directly + order_qt_query_0_after "${query_sql}" + + // create and build complete mv + create_async_mv(db, mv_name, mv_sql) + // refresh mv complete + sql """refresh materialized view ${mv_name} complete""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_0 "select * from ${mv_name}" + + // create and build partition mv + create_async_partition_mv(db, mv_name, mv_sql, "(l_shipdate)") + + // refresh mv partly + sql """refresh materialized view ${mv_name} partitions(p_20231208_20231209)""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_1 "select * from ${mv_name}" + + // query rewrite + mv_rewrite_success(mv_sql, mv_name) + order_qt_query_0_after "${query_sql}" + + // DML + // base table insert into data when not partition table + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'); + """ + sql """refresh materialized view ${mv_name} complete""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_2 "select * from ${mv_name}" + + // base table insert into data when partition table + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'); + """ + sql """refresh materialized view ${mv_name} partitions(p_20231210_20231211)""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_3 "select * from ${mv_name}" +} diff --git a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy index 455ab30d11611f..6eeeff0ae430b6 100644 --- a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy @@ -127,6 +127,9 @@ suite("negative_partition_mv_rewrite") { sql """analyze table lineitem_1 with sync;""" sql """analyze table partsupp_1 with sync;""" + sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');""" + def mv_name = "mv_1" def mtmv_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy index 942db734543330..5bb8c473c03d2a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy @@ -199,6 +199,10 @@ suite("nested_materialized_view") { (2, 3, 10, 11.01, 'supply2'); """ + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // simple nested materialized view def mv1_0_inner_mv = """ select diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index 4dd61a2150cc8c..4870ec99e659c0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -96,12 +96,20 @@ suite("nested_mtmv") { insert into orders_1 values (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'), + (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'), + (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'), (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'), (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'), + (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'), (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'), + (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'), + (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'), (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'), (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'), + (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'), (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'), + (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'), + (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); """ @@ -109,22 +117,29 @@ suite("nested_mtmv") { sql """ insert into lineitem_1 values (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), (1, 1, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), (3, 3, 3, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (3, 3, 3, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'), (1, 2, 3, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), (2, 1, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), + (2, 1, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), (3, 1, 3, 1, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), (1, 2, 1, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), (2, 2, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), (3, 3, 3, 3, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (3, 3, 3, 3, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), (1, 1, 1, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'); """ sql""" insert into partsupp_1 values (1, 1, 1, 99.5, 'yy'), + (1, 1, 1, 99.5, 'yy'), + (2, 2, 2, 109.2, 'mm'), (2, 2, 2, 109.2, 'mm'), (3, 3, 1, 99.5, 'yy'), + (3, 3, 1, 99.5, 'yy'), (3, null, 1, 99.5, 'yy'); """ @@ -132,6 +147,10 @@ suite("nested_mtmv") { sql """analyze table lineitem_1 with sync;""" sql """analyze table partsupp_1 with sync;""" + sql """alter table orders_1 modify column o_orderdate set stats ('row_count'='17');""" + sql """alter table lineitem_1 modify column l_shipdate set stats ('row_count'='14');""" + sql """alter table partsupp_1 modify column ps_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt @@ -659,155 +678,77 @@ suite("nested_mtmv") { left join ( select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 from ( - select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 + select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, + t.agg1 as agg1, + t.sum_total as agg3, + t.max_total as agg4, + t.min_total as agg5, + t.count_all as agg6, + cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, - t.agg1 as agg1, - t.sum_total as agg3, - t.max_total as agg4, - t.min_total as agg5, - t.count_all as agg6, - cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 - from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, - bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 - from lineitem_1 - inner join orders_1 - on lineitem_1.l_orderkey = orders_1.o_orderkey - where lineitem_1.l_shipdate >= "2023-10-17" - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey - ) as t - inner join partsupp_1 - on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey - where partsupp_1.ps_suppkey > 1 - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 - ) as t1 - left join ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, - t.agg1 as agg1, - t.sum_total as agg3, - t.max_total as agg4, - t.min_total as agg5, - t.count_all as agg6, - cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 - from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, - bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 - from lineitem_1 - inner join orders_1 - on lineitem_1.l_orderkey = orders_1.o_orderkey - where lineitem_1.l_shipdate >= "2023-10-17" - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey - ) as t - inner join partsupp_1 - on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey - where partsupp_1.ps_suppkey > 1 - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 - ) as t2 - on t1.l_orderkey = t2.l_orderkey - where t1.l_orderkey > 1 - group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 + select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from lineitem_1 + inner join orders_1 + on lineitem_1.l_orderkey = orders_1.o_orderkey + where lineitem_1.l_shipdate >= "2023-10-17" + group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey + ) as t + inner join partsupp_1 + on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey + where partsupp_1.ps_suppkey > 1 + group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 ) as t1 left join ( - select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 + select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, + t.agg1 as agg1, + t.sum_total as agg3, + t.max_total as agg4, + t.min_total as agg5, + t.count_all as agg6, + cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, - t.agg1 as agg1, - t.sum_total as agg3, - t.max_total as agg4, - t.min_total as agg5, - t.count_all as agg6, - cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 - from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, - bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 - from lineitem_1 - inner join orders_1 - on lineitem_1.l_orderkey = orders_1.o_orderkey - where lineitem_1.l_shipdate >= "2023-10-17" - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey - ) as t - inner join partsupp_1 - on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey - where partsupp_1.ps_suppkey > 1 - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 - ) as t1 - left join ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, - t.agg1 as agg1, - t.sum_total as agg3, - t.max_total as agg4, - t.min_total as agg5, - t.count_all as agg6, - cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 - from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, - bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 - from lineitem_1 - inner join orders_1 - on lineitem_1.l_orderkey = orders_1.o_orderkey - where lineitem_1.l_shipdate >= "2023-10-17" - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey - ) as t - inner join partsupp_1 - on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey - where partsupp_1.ps_suppkey > 1 - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 - ) as t2 - on t1.l_orderkey = t2.l_orderkey - where t1.l_orderkey > 1 - group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 + select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from lineitem_1 + inner join orders_1 + on lineitem_1.l_orderkey = orders_1.o_orderkey + where lineitem_1.l_shipdate >= "2023-10-17" + group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey + ) as t + inner join partsupp_1 + on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey + where partsupp_1.ps_suppkey > 1 + group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 ) as t2 on t1.l_orderkey = t2.l_orderkey + where t1.l_orderkey > 1 group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 ) as t2 on t1.l_orderkey = t2.l_orderkey group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 """ - explain { - sql("${sql_2}") - contains "${mv_2}(${mv_2})" - } + mv_rewrite_any_success(sql_2, [mv_1, mv_2]) compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") - // tmp and will fix soon -// explain { -// sql("${sql_3}") -// contains "${mv_3}(${mv_3})" -// } -// compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") -// -// explain { -// sql("${sql_4}") -// contains "${mv_4}(${mv_4})" -// } -// compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") -// -// explain { -// sql("${sql_5}") -// contains "${mv_5}(${mv_5})" -// } -// compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") + mv_rewrite_any_success(sql_3, [mv_3, mv_4]) + compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") + + mv_rewrite_any_success(sql_4, [mv_3, mv_4]) + compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") + + mv_rewrite_any_success(sql_5, [mv_3, mv_4, mv_5]) + compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") } diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy index f9a84fa5250179..64354267a039bd 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy @@ -99,6 +99,9 @@ suite("nested_mtmv_rewrite_switch") { sql """analyze table orders_2 with sync;""" sql """analyze table lineitem_2 with sync;""" + sql """alter table orders_2 modify column o_orderdate set stats ('row_count'='10');""" + sql """alter table lineitem_2 modify column l_shipdate set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy index 35a983c264c0e2..f8023a9e1e4c49 100644 --- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy @@ -480,6 +480,10 @@ suite("partition_mv_rewrite") { analyze table orders with sync; """ + sql """alter table orders modify column o_comment set stats ('row_count'='3');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='6');""" + sql """alter table lineitem_static modify column l_comment set stats ('row_count'='4');""" + // should rewrite successful when union rewrite enalbe if base table add new partition mv_rewrite_success(roll_up_all_partition_sql, "mv_10086", true, is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) diff --git a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy index 6b5c2dc45e875f..c42888e35abc0a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy @@ -123,6 +123,10 @@ suite("mv_scan_table") { sql """analyze table lineitem with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" +sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // with filter def mv1_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy index dfbe433fb9b3b0..24122f18da52e4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy @@ -179,6 +179,9 @@ suite("union_all_compensate") { mv_rewrite_fail(query1_0, "test_agg_mv") order_qt_query1_1_after "${query1_0}" + sql """alter table test_table1 modify column num set stats ('row_count'='20');""" + sql """alter table test_table2 modify column num set stats ('row_count'='16');""" + // Aggregate, if query group by expression doesn't use the partition column, but the invalid partition is in the // grace_period, should not compensate union all, but should rewritten successfully diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy index 2b00c8200d2589..d1bdba31794acb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy @@ -78,6 +78,9 @@ suite ("partition_curd_union_rewrite") { ); """ + sql """alter table orders modify column o_comment set stats ('row_count'='3');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='3');""" + sql""" insert into orders values (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'), diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy index 702431e5e391ae..4e74090b1a2bb2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy @@ -90,6 +90,9 @@ suite ("usercase_union_rewrite") { sql """analyze table orders_user with sync;""" sql """analyze table lineitem_user with sync;""" + sql """alter table orders_user modify column o_comment set stats ('row_count'='4');""" + sql """alter table lineitem_user modify column l_comment set stats ('row_count'='3');""" + def create_mv_orders = { mv_name, mv_sql -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};""" sql """DROP TABLE IF EXISTS ${mv_name}""" diff --git a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy index 5c64379950f942..cba2509483b6dc 100644 --- a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy @@ -57,6 +57,8 @@ suite("null_unsafe_equals") { (5, 2, 'o', 1.2, '2023-12-12', 'c','d', null, 'mi'); """ + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + def mv1_0 = """ select count(*), o_orderstatus, o_comment diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy index 62da95cf8c0149..9dddde1098f684 100644 --- a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy @@ -87,6 +87,9 @@ suite("variant_mv") { sql """analyze table github_events1 with sync;""" sql """analyze table github_events2 with sync;""" + sql """alter table github_events1 modify column created_at set stats ('row_count'='3');""" + sql """alter table github_events2 modify column created_at set stats ('row_count'='3');""" + // variant appear in where both slot and in expression def mv1_0 = """ SELECT diff --git a/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy b/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy index 400e113b6f3ae5..c51dea2f5a7403 100644 --- a/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy @@ -112,6 +112,9 @@ suite("with_select_table_auth","p0,auth") { sql """analyze table lineitem with sync""" sql """analyze table orders with sync""" + sql """alter table orders modify column o_comment set stats ('row_count'='18');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """grant select_priv on ${db}.orders to ${user_name}""" sql """grant select_priv on ${db}.lineitem to ${user_name}""" sql """grant select_priv on regression_test to ${user_name}""" diff --git a/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy b/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy index b7c6ecbd8ae008..757a4c42e5ad32 100644 --- a/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy @@ -143,6 +143,10 @@ suite("query_with_sql_limit") { sql """analyze table lineitem with sync""" sql """analyze table orders with sync""" + sql """alter table orders modify column o_comment set stats ('row_count'='18');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='3');""" + // test sql_select_limit default, default 9223372036854775807 sql """set sql_select_limit = 2;""" def mv1_0 = diff --git a/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy b/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy new file mode 100644 index 00000000000000..5ba2c05053e2a2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("with_table_operator") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders; + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + drop table if exists orders_partition; + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders_partition ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders_partition values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + create_async_mv(db, "complete_agg_mv", """ + select count(*),o_orderdate + from orders + group by o_orderdate; + """) + + create_async_mv(db, "complete_mv", """ + select * from orders; + """) + + create_async_partition_mv(db, "partition_agg_mv", """ + select count(*),o_orderdate + from orders_partition + group by o_orderdate; + """, "(o_orderdate)") + + create_async_partition_mv(db, "partition_mv", """ + select * from orders_partition; + """, "(o_orderdate)") + + // query rewrite with tablet should fail + mv_rewrite_all_fail(""" + select count(*),o_orderdate + from orders TABLET(110) + group by o_orderdate; + """, ["complete_mv", "complete_agg_mv"]) + + mv_rewrite_all_fail("""select * from orders TABLET(110);""", ["complete_mv", "complete_agg_mv"]) + + // query rewrite with index should fail + createMV(""" + create materialized view query_index_test + as + select o_orderdate, count(*) + from orders + group by o_orderdate; + """) + mv_rewrite_all_fail(""" + select * from orders index query_index_test; + """, ["complete_mv", "complete_agg_mv"]) + + // query rewrite with sample should fail + mv_rewrite_all_fail(""" + select count(*),o_orderdate + from orders TABLESAMPLE(20 percent) + group by o_orderdate; + """, ["complete_mv", "complete_agg_mv"]) + + mv_rewrite_all_fail("""select * from orders TABLESAMPLE(20 percent);""", ["complete_mv", "complete_agg_mv"]) + + // query rewrite with partition should fail + mv_rewrite_all_fail(""" + select count(*),o_orderdate + from orders_partition PARTITION (day_2) + group by o_orderdate; + """, ["partition_mv", "partition_agg_mv"]) + + mv_rewrite_all_fail("""select * from orders_partition PARTITION (day_2);""", ["partition_mv", "partition_agg_mv"]) + + + sql """ + DROP MATERIALIZED VIEW IF EXISTS complete_agg_mv; + """ + sql """ + DROP MATERIALIZED VIEW IF EXISTS partition_agg_mv; + """ + sql """ + DROP MATERIALIZED VIEW IF EXISTS complete_mv; + """ + sql """ + DROP MATERIALIZED VIEW IF EXISTS partition_mv; + """ + sql """ + DROP MATERIALIZED VIEW IF EXISTS query_index_test on orders; + """ +} + diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy index 4373e0b7073284..5cbdf2994e5a5f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy @@ -36,6 +36,7 @@ suite ("agg_have_dup_base") { distributed BY hash(k1) buckets 3 properties("replication_num" = "1"); """ + sql """alter table agg_have_dup_base modify column k1 set stats ('row_count'='5');""" sql "insert into agg_have_dup_base select 1,1,1,'a';" sql "insert into agg_have_dup_base select 2,2,2,'b';" diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy index 91293bb747aa64..e2f334566e9c24 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy @@ -36,6 +36,7 @@ suite ("case_ignore") { sql "insert into case_ignore select 2,2,2,'b';" sql "insert into case_ignore select 3,-3,null,'c';" + createMV ("create materialized view k12a as select K1,abs(K2) from case_ignore;") sleep(3000) @@ -55,6 +56,7 @@ suite ("case_ignore") { order_qt_select_mv "select K1,abs(K2) from case_ignore order by K1;" sql """set enable_stats=true;""" + sql """alter table case_ignore modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,abs(k2) from case_ignore order by k1;", "k12a") mv_rewrite_success("select K1,abs(K2) from case_ignore order by K1;", "k12a") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy index 132a4c3f054571..3537777d312c58 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy @@ -57,6 +57,7 @@ suite ("dup_gb_mv_abs") { order_qt_select_mv_sub "select sum(abs(k2)) from dup_gb_mv_abs group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table dup_gb_mv_abs modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,sum(abs(k2)) from dup_gb_mv_abs group by k1;", "k12sa") mv_rewrite_success("select sum(abs(k2)) from dup_gb_mv_abs group by k1;", "k12sa") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy index 45271415b98ef5..2e6e9c6b627850 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy @@ -57,6 +57,7 @@ suite ("dup_gb_mv_plus") { order_qt_select_mv_sub "select sum(k2+1) from dup_gb_mv_plus group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table dup_gb_mv_plus modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,sum(k2+1) from dup_gb_mv_plus group by k1;", "k12sp") mv_rewrite_success("select sum(k2+1) from dup_gb_mv_plus group by k1;", "k12sp") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy index c4770d6dcd0262..6cef52fd96edc7 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy @@ -31,7 +31,6 @@ suite ("dup_mv_abs") { distributed BY hash(k1) buckets 3 properties("replication_num" = "1"); """ - sql "insert into dup_mv_abs select 1,1,1,'a';" sql "insert into dup_mv_abs select 2,2,2,'b';" sql "insert into dup_mv_abs select 3,-3,null,'c';" @@ -69,6 +68,7 @@ suite ("dup_mv_abs") { order_qt_select_group_mv_not "select sum(abs(k2)) from dup_mv_abs group by k3 order by k3;" sql """set enable_stats=true;""" + sql """alter table dup_mv_abs modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,abs(k2) from dup_mv_abs order by k1;", "k12a") mv_rewrite_success("select abs(k2) from dup_mv_abs order by k1;", "k12a") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy index 788647ae224d98..7fd22f5b71892c 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy @@ -69,6 +69,7 @@ suite ("dup_mv_bin") { order_qt_select_group_mv_not "select group_concat(bin(k2)) from dup_mv_bin group by k3 order by k3;" sql """set enable_stats=true;""" + sql """alter table dup_mv_bin modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,bin(k2) from dup_mv_bin order by k1;", "k12b") mv_rewrite_success("select bin(k2) from dup_mv_bin order by k1;", "k12b") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy index d049a0aeb7c1b0..f5784e7a5fdaa6 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy @@ -45,6 +45,7 @@ suite ("dup_mv_bm_hash") { order_qt_select_mv "select bitmap_union_count(to_bitmap(k2)) from dup_mv_bm_hash group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table dup_mv_bm_hash modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from dup_mv_bm_hash group by k1 order by k1;", "dup_mv_bm_hash_mv1") createMV("create materialized view dup_mv_bm_hash_mv2 as select k1,bitmap_union(bitmap_hash(k3)) from dup_mv_bm_hash group by k1;") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy index 0b7dce804b2504..77a12a3768ae87 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy @@ -46,7 +46,6 @@ suite ("dup_mv_plus") { sql "analyze table dup_mv_plus with sync;" sql """set enable_stats=false;""" - order_qt_select_star "select * from dup_mv_plus order by k1;" mv_rewrite_success("select k1,k2+1 from dup_mv_plus order by k1;", "k12p") @@ -87,6 +86,7 @@ suite ("dup_mv_plus") { order_qt_select_mv "select k1,k2+1 from dup_mv_plus order by k2;" sql """set enable_stats=true;""" + sql """alter table dup_mv_plus modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,k2+1 from dup_mv_plus order by k1;", "k12p") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy index c5098c55074c7d..95902c27866481 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy @@ -45,6 +45,7 @@ suite ("dup_mv_year") { order_qt_select_mv "select k1,year(k2) from dup_mv_year order by k1;" sql """set enable_stats=true;""" + sql """alter table dup_mv_year modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,year(k2) from dup_mv_year order by k1;", "k12y") createMV "create materialized view k13y as select k1,year(k3) from dup_mv_year;" diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy index 7e317d501bea07..d0fa214e98f042 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy @@ -52,5 +52,6 @@ suite ("multi_slot1") { order_qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1 order by abs(k1)+k2+1,abs(k2+2)+k3+3;" sql """set enable_stats=true;""" + sql """alter table multi_slot1 modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1 order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy index 6bbded534a77ee..0fa36b77b44114 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy @@ -65,6 +65,7 @@ suite ("multi_slot2") { order_qt_select_base "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2 order by abs(k1)+k2;" sql """set enable_stats=true;""" + sql """alter table multi_slot2 modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2+1 order by abs(k1)+k2+1", "k1a2p2ap3ps") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy index c6016296e8b352..51cc90aaf6ae26 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy @@ -53,5 +53,6 @@ suite ("multi_slot3") { order_qt_select_mv "select k1+1,abs(k2+2)+k3+3 from multi_slot3 order by k1+1;" sql """set enable_stats=true;""" + sql """alter table multi_slot3 modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1+1,abs(k2+2)+k3+3 from multi_slot3 order by k1+1;", "k1p2ap3p") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy index f8210f914060f7..1112b93d0a6d2f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy @@ -55,5 +55,6 @@ suite ("multi_slot4") { order_qt_select_mv "select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1 order by k1+1;" sql """set enable_stats=true;""" + sql """alter table multi_slot4 modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1 order by k1+1;", "k1p2ap3ps") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy index e10bed2dced7ef..7fa2e7ba6cda6f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy @@ -63,5 +63,6 @@ suite ("multi_slot5") { order_qt_select_mv "select k1,version() from multi_slot5 order by k1;" sql """set enable_stats=true;""" + sql """alter table multi_slot5 modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k2+k3 from multi_slot5 order by k1;", "k123p") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy index 9201a1cea4d321..2ed66fc8477a37 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy @@ -84,5 +84,6 @@ suite ("multi_slot6") { order_qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6 order by abs(k1)+k2+1,abs(k2+2)+k3+3;" sql """set enable_stats=true;""" + sql """alter table multi_slot6 modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6 order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy index d6cd0d5a24421a..387c155a5e45c8 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy @@ -47,7 +47,6 @@ suite ("single_slot") { sql "analyze table single_slot with sync;" sql """set enable_stats=false;""" - order_qt_select_star "select * from single_slot order by k1;" explain { @@ -58,5 +57,6 @@ suite ("single_slot") { order_qt_select_mv "select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t;" sql """set enable_stats=true;""" + sql """alter table single_slot modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t;", "k1ap2spa") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy index 031560a41ef502..be8bc50532ff81 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy @@ -62,7 +62,7 @@ suite ("sum_devide_count") { order_qt_select_mv "select sum(k2)/count(k2) from sum_devide_count;" sql """set enable_stats=true;""" - + sql """alter table sum_devide_count modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from sum_devide_count group by k1,k4 order by k1,k4;", "kavg") mv_rewrite_success("select k1,sum(k2)/count(k2) from sum_devide_count group by k1 order by k1;", "kavg") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy index df390b26551054..1ebaa8884ab383 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy @@ -47,6 +47,7 @@ suite ("unique_mv") { mv_rewrite_success("SELECT * FROM c5816_t WHERE call_uuid='adc';", "mv_1") sql """set enable_stats=true;""" + sql """alter table c5816_t modify column org_id set stats ('row_count'='1');""" mv_rewrite_success("SELECT * FROM c5816_t WHERE call_uuid='adc';", "mv_1") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy index 9a5e42034be3cb..2b6b0dd354e4cd 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy @@ -37,6 +37,7 @@ suite ("MVMultiUsage") { sql """insert into MVMultiUsage values("2020-01-02",2,"b",2,2,2);""" sql """insert into MVMultiUsage values("2020-01-03",3,"c",3,3,3);""" + createMV("create materialized view MVMultiUsage_mv as select deptno, empid, salary from MVMultiUsage order by deptno;") sleep(3000) @@ -58,6 +59,8 @@ suite ("MVMultiUsage") { order_qt_select_mv "select * from (select deptno, empid from MVMultiUsage where deptno>100) A join (select deptno, empid from MVMultiUsage where deptno >200) B using (deptno) order by 1;" sql """set enable_stats=true;""" + sql """alter table MVMultiUsage modify column time_col set stats ('row_count'='4');""" + mv_rewrite_fail("select * from MVMultiUsage order by empid;", "MVMultiUsage_mv") explain { sql("select * from (select deptno, empid from MVMultiUsage where deptno>100) A join (select deptno, empid from MVMultiUsage where deptno >200) B using (deptno);") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy index 246c97022e9e85..86a85dcc647443 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy @@ -30,6 +30,7 @@ suite ("MVWithAs") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into MVWithAs values("2020-01-01",1,"a",1);""" sql """insert into MVWithAs values("2020-01-01",1,"a",1);""" sql """insert into MVWithAs values("2020-01-01",1,"a",1);""" @@ -53,6 +54,8 @@ suite ("MVWithAs") { order_qt_select_mv "select count(tag_id) from MVWithAs t;" sql """set enable_stats=true;""" + sql """alter table MVWithAs modify column time_col set stats ('row_count'='7');""" + mv_rewrite_fail("select * from MVWithAs order by time_col;", "MVWithAs_mv") mv_rewrite_success("select count(tag_id) from MVWithAs t;", "MVWithAs_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy index 9500c6ff1919ae..b48db921cc9a5d 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy @@ -31,6 +31,8 @@ suite ("aggCDInBitmap") { sql """insert into aggCDInBitmap values(2,to_bitmap(2));""" sql """insert into aggCDInBitmap values(3,to_bitmap(3));""" + sql "analyze table aggCDInBitmap with sync;" + sql """set enable_stats=false;""" order_qt_select_star "select * from aggCDInBitmap order by 1;" @@ -41,4 +43,11 @@ suite ("aggCDInBitmap") { } order_qt_select_mv "select k1, count(distinct v1) from aggCDInBitmap group by k1 order by k1;" + sql """set enable_stats=true;""" + sql """alter table aggCDInBitmap modify column k1 set stats ('row_count'='3');""" + explain { + sql("select k1, count(distinct v1) from aggCDInBitmap group by k1;") + contains "bitmap_union_count" + } + } diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy index 1a7ccd851503d8..a234661150e732 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy @@ -38,7 +38,6 @@ suite ("aggMVCalcAggFun") { sql """insert into aggMVCalcAggFun values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggMVCalcAggFun values("2020-01-03",3,"c",3,3,3);""" - createMV("create materialized view aggMVCalcAggFunMv as select deptno, empid, sum(salary) from aggMVCalcAggFun group by empid, deptno;") sleep(3000) @@ -55,6 +54,7 @@ suite ("aggMVCalcAggFun") { order_qt_select_mv "select deptno, sum(salary + 1) from aggMVCalcAggFun where deptno > 10 group by deptno order by deptno;" sql """set enable_stats=true;""" + sql """alter table aggMVCalcAggFun modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggMVCalcAggFun order by empid;", "aggMVCalcAggFunMv")\ mv_rewrite_fail("select deptno, sum(salary + 1) from aggMVCalcAggFun where deptno > 10 group by deptno;", "aggMVCalcAggFunMv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy index 0cb12cbcde0b51..c29eec40c8da93 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy @@ -54,6 +54,7 @@ suite ("aggOnAggMV1") { order_qt_select_mv "select sum(salary), deptno from aggOnAggMV1 group by deptno order by deptno;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV1 modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggOnAggMV1 order by empid;", "aggOnAggMV1_mv") mv_rewrite_success("select sum(salary), deptno from aggOnAggMV1 group by deptno order by deptno;", "aggOnAggMV1_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy index bd826537ff88f7..e38a27c1db3c53 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy @@ -54,6 +54,7 @@ suite ("aggOnAggMV10") { order_qt_select_mv "select deptno, commission, sum(salary) + 1 from aggOnAggMV10 group by rollup (deptno, commission) order by 1,2;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV10 modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggOnAggMV10 order by empid;", "aggOnAggMV10_mv") mv_rewrite_success("select deptno, commission, sum(salary) + 1 from aggOnAggMV10 group by rollup (deptno, commission);", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy index 793389ec68e4ae..b0f379b19476b8 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy @@ -33,6 +33,7 @@ suite ("aggOnAggMV11") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into aggOnAggMV11 values("2020-01-01",1,"a",1,1,1);""" sql """insert into aggOnAggMV11 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV11 values("2020-01-03",3,"c",3,3,3);""" @@ -54,6 +55,8 @@ suite ("aggOnAggMV11") { order_qt_select_mv "select deptno, count(salary) + count(1) from aggOnAggMV11 group by deptno order by 1;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV11 modify column time_col set stats ('row_count'='4');""" + mv_rewrite_fail("select * from aggOnAggMV11 order by empid;", "aggOnAggMV11_mv") mv_rewrite_fail("select deptno, count(salary) + count(1) from aggOnAggMV11 group by deptno;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy index 32d897939ea652..179e22584d32a1 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy @@ -56,6 +56,7 @@ suite ("aggOnAggMV2") { order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV2 group by deptno) a where (sum_salary * 2) > 3 order by deptno ;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV2 modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from aggOnAggMV2 order by empid;", "aggOnAggMV2_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV2 group by deptno) a where (sum_salary * 2) > 3 order by deptno ;", "aggOnAggMV2_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy index e03c6d8b195a68..6816e163261986 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy @@ -55,6 +55,7 @@ suite ("aggOnAggMV3") { order_qt_select_mv "select commission, sum(salary) from aggOnAggMV3 where commission * (deptno + commission) = 100 group by commission order by commission;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV3 modify column time_col set stats ('row_count'='5');""" mv_rewrite_fail("select * from aggOnAggMV3 order by empid;", "aggOnAggMV3_mv") mv_rewrite_success("select commission, sum(salary) from aggOnAggMV3 where commission * (deptno + commission) = 100 group by commission order by commission;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy index 481d8f19b5a133..d90057b8bf5120 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy @@ -33,6 +33,8 @@ suite ("aggOnAggMV5") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """alter table aggOnAggMV5 modify column time_col set stats ('row_count'='4');""" + sql """insert into aggOnAggMV5 values("2020-01-01",1,"a",1,1,1);""" sql """insert into aggOnAggMV5 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV5 values("2020-01-03",3,"c",3,3,3);""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy index abd66e1ab13ccd..7e2e87718b8e76 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy @@ -54,6 +54,7 @@ suite ("aggOnAggMV6") { order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV6 where deptno>=20 group by deptno) a where sum_salary>10 order by 1;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV6 modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggOnAggMV6 order by empid;", "aggOnAggMV6_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV6 where deptno>=20 group by deptno) a where sum_salary>10;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy index 53b75d03d85167..386871a8a5a98f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy @@ -54,6 +54,7 @@ suite ("aggOnAggMV7") { order_qt_select_mv "select deptno, sum(salary) from aggOnAggMV7 where deptno>=20 group by deptno order by 1;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV7 modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggOnAggMV7 order by empid;", "aggOnAggMV7_mv") mv_rewrite_success("select deptno, sum(salary) from aggOnAggMV7 where deptno>=20 group by deptno;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy index 13e2097cdb46ba..8b2697cb1f78f7 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy @@ -30,6 +30,7 @@ suite ("bitmapUnionIn") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into bitmapUnionIn values("2020-01-01",1,"a",1);""" sql """insert into bitmapUnionIn values("2020-01-02",2,"b",2);""" @@ -50,6 +51,8 @@ suite ("bitmapUnionIn") { order_qt_select_mv "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from bitmapUnionIn group by user_id having a>1 order by a;" sql """set enable_stats=true;""" + sql """alter table bitmapUnionIn modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from bitmapUnionIn order by time_col;", "bitmapUnionIn_mv") mv_rewrite_success("select user_id, bitmap_union_count(to_bitmap(tag_id)) a from bitmapUnionIn group by user_id having a>1 order by a;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy index c28dac1d227aab..52abff46678bb0 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy @@ -31,6 +31,7 @@ suite ("incMVReInSub") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into incMVReInSub values("2020-01-01",1,"a",1);""" sql """insert into incMVReInSub values("2020-01-02",2,"b",2);""" @@ -52,6 +53,8 @@ suite ("incMVReInSub") { order_qt_select_mv "select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub where user_name in (select user_name from incMVReInSub group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table incMVReInSub modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from incMVReInSub order by time_col;", "incMVReInSub_mv") mv_rewrite_fail("select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub where user_name in (select user_name from incMVReInSub group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy index ee8e688c0e7313..6172773ca7a43b 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy @@ -31,6 +31,7 @@ suite ("incRewriteCD") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into incRewriteCD values("2020-01-01",1,"a",1);""" sql """insert into incRewriteCD values("2020-01-02",2,"b",2);""" @@ -50,6 +51,8 @@ suite ("incRewriteCD") { order_qt_select_mv "select user_name, count(distinct tag_id) from incRewriteCD group by user_name order by user_name;" sql """set enable_stats=true;""" + sql """alter table incRewriteCD modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from incRewriteCD order by time_col;", "incRewriteCD_mv") mv_rewrite_fail("select user_name, count(distinct tag_id) from incRewriteCD group by user_name;", "incRewriteCD_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy index f2f7a014d98fc8..69d1c39931c17f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy @@ -31,6 +31,8 @@ suite ("joinOnLeftPToJoin") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """alter table joinOnLeftPToJoin modify column time_col set stats ('row_count'='3');""" + sql """insert into joinOnLeftPToJoin values("2020-01-02",2,"b",2,2,2);""" sql """insert into joinOnLeftPToJoin values("2020-01-03",3,"c",3,3,3);""" sql """insert into joinOnLeftPToJoin values("2020-01-02",2,"b",2,7,2);""" @@ -45,6 +47,7 @@ suite ("joinOnLeftPToJoin") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into joinOnLeftPToJoin_1 values("2020-01-02",2,"b",2);""" sql """insert into joinOnLeftPToJoin_1 values("2020-01-03",3,"c",3);""" sql """insert into joinOnLeftPToJoin_1 values("2020-01-02",2,"b",1);""" @@ -64,6 +67,8 @@ suite ("joinOnLeftPToJoin") { order_qt_select_mv "select * from (select deptno , sum(salary) from joinOnLeftPToJoin group by deptno) A join (select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno ) B on A.deptno = B.deptno order by A.deptno;" sql """set enable_stats=true;""" + sql """alter table joinOnLeftPToJoin_1 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_all_success("select * from (select deptno , sum(salary) from joinOnLeftPToJoin group by deptno) A join (select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno ) B on A.deptno = B.deptno;", ["joinOnLeftPToJoin_mv", "joinOnLeftPToJoin_1_mv"]) } diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy index 7f455da429e6c3..21864f973ee8c0 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy @@ -54,6 +54,8 @@ suite ("orderByOnPView") { order_qt_select_mv "select empid from orderByOnPView order by deptno;" sql """set enable_stats=true;""" + sql """alter table orderByOnPView modify column time_col set stats ('row_count'='4');""" + mv_rewrite_fail("select * from orderByOnPView where time_col='2020-01-01' order by empid;", "orderByOnPView_mv") mv_rewrite_success("select empid from orderByOnPView where deptno = 0 order by deptno;", "orderByOnPView_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy index 384137471d827f..157a15498e717c 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy @@ -33,6 +33,7 @@ suite ("projectMV1") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into projectMV1 values("2020-01-01",1,"a",1,1,1);""" sql """insert into projectMV1 values("2020-01-02",2,"b",2,2,2);""" @@ -52,6 +53,8 @@ suite ("projectMV1") { order_qt_select_mv "select empid, deptno from projectMV1 order by empid;" sql """set enable_stats=true;""" + sql """alter table projectMV1 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from projectMV1 where time_col='2020-01-01' order by empid;", "projectMV1_mv") mv_rewrite_success("select empid, deptno from projectMV1 where deptno=0 order by empid;", "projectMV1_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy index 672bda198b01aa..4d85fcc9c8aeae 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy @@ -33,6 +33,7 @@ suite ("projectMV2") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into projectMV2 values("2020-01-01",1,"a",1,1,1);""" sql """insert into projectMV2 values("2020-01-02",2,"b",2,2,2);""" @@ -55,6 +56,8 @@ suite ("projectMV2") { order_qt_select_base "select name from projectMV2 where deptno -1 = 0 order by empid;" sql """set enable_stats=true;""" + sql """alter table projectMV2 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from projectMV2 order by empid;", "projectMV2_mv") mv_rewrite_success("select empid + 1 from projectMV2 where deptno = 1 order by empid;", "projectMV2_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy index 37b73a9a81aff4..9ad301fbbc6a97 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy @@ -33,6 +33,7 @@ suite ("projectMV3") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into projectMV3 values("2020-01-01",1,"a",1,1,1);""" sql """insert into projectMV3 values("2020-01-02",2,"b",2,2,2);""" @@ -57,6 +58,8 @@ suite ("projectMV3") { order_qt_select_mv2 "select name from projectMV3 where deptno -1 = 0 order by empid;" sql """set enable_stats=true;""" + sql """alter table projectMV3 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from projectMV3 order by empid;", "projectMV3_mv") mv_rewrite_success("select empid + 1, name from projectMV3 where deptno = 1 order by empid;", "projectMV3_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy index ea87551dfa1249..ddcdada549ad8f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy @@ -33,6 +33,7 @@ suite ("projectMV4") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into projectMV4 values("2020-01-01",1,"a",1,1,1);""" sql """insert into projectMV4 values("2020-01-02",2,"b",2,2,2);""" @@ -57,6 +58,8 @@ suite ("projectMV4") { order_qt_select_base "select empid from projectMV4 where deptno > 1 and empid > 1 order by empid;" sql """set enable_stats=true;""" + sql """alter table projectMV4 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from projectMV4 order by empid;", "projectMV4_mv") mv_rewrite_success("select name from projectMV4 where deptno > 1 and salary > 1 and name = 'a' order by name;", "projectMV4_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy index 25300c9d602681..67312eed088ed4 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy @@ -37,7 +37,6 @@ suite ("subQuery") { sql """insert into subQuery values("2020-01-02",2,"b",2,2,2);""" sql """insert into subQuery values("2020-01-03",3,"c",3,3,3);""" - createMV("create materialized view subQuery_mv as select deptno, empid from subQuery;") sleep(3000) @@ -60,6 +59,7 @@ suite ("subQuery") { */ sql """set enable_stats=true;""" + sql """alter table subQuery modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from subQuery order by empid;", "subQuery_mv") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy index 0f57b5c6c1687e..9feb1aa1c975f5 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy @@ -58,6 +58,8 @@ suite ("unionDis") { order_qt_select_mv "select * from (select empid, deptno from unionDis where empid >1 union select empid, deptno from unionDis where empid <0) t order by 1;" sql """set enable_stats=true;""" + sql """alter table unionDis modify column time_col set stats ('row_count'='4');""" + mv_rewrite_fail("select * from unionDis order by empid;", "unionDis_mv") explain { diff --git a/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy b/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy index ceada782a27788..ca429313770757 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy @@ -41,6 +41,7 @@ suite("agg") { AGGREGATE KEY (siteid,citycode,username) DISTRIBUTED BY HASH(siteid) BUCKETS 5 properties("replication_num" = "1"); """ + sql """alter table test_rollup_agg1 modify column siteid set stats ('row_count'='3');""" sql """ALTER TABLE ${tbName} ADD ROLLUP rollup_city(citycode, pv);""" int max_try_secs = 60 while (max_try_secs--) { diff --git a/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy b/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy index b711728f769129..07af0da35098e1 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy @@ -44,6 +44,7 @@ suite("agg_date", "rollup") { AGGREGATE KEY (datek1, datetimek1, datetimek2, datetimek3) DISTRIBUTED BY HASH(datek1) BUCKETS 5 properties("replication_num" = "1"); """ + sql """alter table test_rollup_agg_date1 modify column datek1 set stats ('row_count'='2');""" sql """ALTER TABLE ${tbName} ADD ROLLUP rollup_date(datek1,datetimek2,datetimek1,datetimek3,datev1,datetimev1,datetimev2,datetimev3);""" int max_try_secs = 60 while (max_try_secs--) { diff --git a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy index a3917dee28a75a..2407c70d8a451c 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy @@ -34,6 +34,7 @@ suite("bitmap", "rollup") { ) DISTRIBUTED BY HASH(k1) properties("replication_num" = "1"); """ + sql """alter table test_materialized_view_bitmap1 modify column k1 set stats ('row_count'='2');""" sql "CREATE MATERIALIZED VIEW test_neg as select k1,bitmap_union(to_bitmap(k2)), bitmap_union(to_bitmap(k3)) FROM ${tbName1} GROUP BY k1;" max_try_secs = 60 diff --git a/regression-test/suites/nereids_syntax_p0/rollup/date.groovy b/regression-test/suites/nereids_syntax_p0/rollup/date.groovy index e1eb0ca34418e1..34d5e338cfa406 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/date.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/date.groovy @@ -126,6 +126,7 @@ suite("date", "rollup") { mv_rewrite_success("SELECT store_id, max(sale_datetime3) FROM ${tbName1} GROUP BY store_id", "amt_max4") sql """set enable_stats=true;""" + sql """alter table test_materialized_view_date1 modify column record_id set stats ('row_count'='2');""" mv_rewrite_success("SELECT store_id, max(sale_date1) FROM ${tbName1} GROUP BY store_id", "amt_max1") mv_rewrite_success("SELECT store_id, max(sale_datetime1) FROM ${tbName1} GROUP BY store_id", "amt_max2") diff --git a/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy b/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy index 2ba3540832945a..5002c509a20617 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy @@ -46,6 +46,7 @@ suite("hll", "rollup") { "amt_count") sql """set enable_stats=true;""" + sql """alter table test_materialized_view_hll1 modify column record_id set stats ('row_count'='2');""" mv_rewrite_success("SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM test_materialized_view_hll1 GROUP BY store_id;", "amt_count") } diff --git a/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy b/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy index 51599f294d1e62..d7dec7696d95eb 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy @@ -43,6 +43,7 @@ suite("hll_with_light_sc", "rollup") { "amt_count1") sql """set enable_stats=true;""" + sql """alter table test_materialized_view_hll_with_light_sc1 modify column record_id set stats ('row_count'='2');""" mv_rewrite_success("SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM test_materialized_view_hll_with_light_sc1 GROUP BY store_id;", "amt_count1") }