diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 47cf0b7b458d84f..6405967d9d9b765 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -17,16 +17,9 @@ package org.apache.doris.nereids; -import org.apache.doris.catalog.DatabaseIf; -import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; -import org.apache.doris.datasource.CatalogIf; import org.apache.doris.nereids.analyzer.Scope; -import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; -import org.apache.doris.nereids.analyzer.UnboundRelation; -import org.apache.doris.nereids.analyzer.UnboundTableSink; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.hint.Hint; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.nereids.jobs.JobContext; @@ -54,13 +47,8 @@ import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.ColumnStatistic; @@ -407,159 +395,6 @@ private CascadesContext execute(Job job) { return this; } - /** - * Extract tables. - */ - public void extractTables(LogicalPlan logicalPlan) { - Set> tableNames = getTables(logicalPlan); - tables = Maps.newHashMap(); - for (List tableName : tableNames) { - try { - TableIf table = getTable(tableName); - tables.put(table.getFullQualifiers(), table); - } catch (Throwable e) { - // IGNORE - } - } - - } - - public Map, TableIf> getTables() { - if (tables == null) { - return null; - } else { - return tables; - } - } - - private Set> getTables(LogicalPlan logicalPlan) { - final Set> tableNames = new HashSet<>(); - logicalPlan.foreach(p -> { - if (p instanceof LogicalFilter) { - tableNames.addAll(extractTableNamesFromFilter((LogicalFilter) p)); - } else if (p instanceof LogicalCTE) { - tableNames.addAll(extractTableNamesFromCTE((LogicalCTE) p)); - } else if (p instanceof LogicalProject) { - tableNames.addAll(extractTableNamesFromProject((LogicalProject) p)); - } else if (p instanceof LogicalHaving) { - tableNames.addAll(extractTableNamesFromHaving((LogicalHaving) p)); - } else if (p instanceof UnboundOneRowRelation) { - tableNames.addAll(extractTableNamesFromOneRowRelation((UnboundOneRowRelation) p)); - } else { - Set logicalPlans = p.collect( - n -> (n instanceof UnboundRelation || n instanceof UnboundTableSink)); - for (LogicalPlan plan : logicalPlans) { - if (plan instanceof UnboundRelation) { - tableNames.add(((UnboundRelation) plan).getNameParts()); - } else if (plan instanceof UnboundTableSink) { - tableNames.add(((UnboundTableSink) plan).getNameParts()); - } else { - throw new AnalysisException("get tables from plan failed. meet unknown type node " + plan); - } - } - } - }); - return tableNames; - } - - private Set> extractTableNamesFromHaving(LogicalHaving having) { - Set subqueryExprs = having.getPredicate() - .collect(SubqueryExpr.class::isInstance); - Set> tableNames = new HashSet<>(); - for (SubqueryExpr expr : subqueryExprs) { - LogicalPlan plan = expr.getQueryPlan(); - tableNames.addAll(getTables(plan)); - } - return tableNames; - } - - private Set> extractTableNamesFromOneRowRelation(UnboundOneRowRelation oneRowRelation) { - Set subqueryExprs = oneRowRelation.getProjects().stream() - .>map(p -> p.collect(SubqueryExpr.class::isInstance)) - .flatMap(Set::stream) - .collect(Collectors.toSet()); - Set> tableNames = new HashSet<>(); - for (SubqueryExpr expr : subqueryExprs) { - LogicalPlan plan = expr.getQueryPlan(); - tableNames.addAll(getTables(plan)); - } - return tableNames; - } - - private Set> extractTableNamesFromProject(LogicalProject project) { - Set subqueryExprs = project.getProjects().stream() - .>map(p -> p.collect(SubqueryExpr.class::isInstance)) - .flatMap(Set::stream) - .collect(Collectors.toSet()); - Set> tableNames = new HashSet<>(); - for (SubqueryExpr expr : subqueryExprs) { - LogicalPlan plan = expr.getQueryPlan(); - tableNames.addAll(getTables(plan)); - } - return tableNames; - } - - private Set> extractTableNamesFromFilter(LogicalFilter filter) { - Set subqueryExprs = filter.getPredicate() - .collect(SubqueryExpr.class::isInstance); - Set> tableNames = new HashSet<>(); - for (SubqueryExpr expr : subqueryExprs) { - LogicalPlan plan = expr.getQueryPlan(); - tableNames.addAll(getTables(plan)); - } - return tableNames; - } - - private Set> extractTableNamesFromCTE(LogicalCTE cte) { - List> subQueryAliases = cte.getAliasQueries(); - Set> tableNames = new HashSet<>(); - for (LogicalSubQueryAlias subQueryAlias : subQueryAliases) { - tableNames.addAll(getTables(subQueryAlias)); - } - return tableNames; - } - - private TableIf getTable(List nameParts) { - switch (nameParts.size()) { - case 1: { // table - String ctlName = getConnectContext().getEnv().getCurrentCatalog().getName(); - String dbName = getConnectContext().getDatabase(); - return getTable(ctlName, dbName, nameParts.get(0), getConnectContext().getEnv()); - } - case 2: { // db.table - String ctlName = getConnectContext().getEnv().getCurrentCatalog().getName(); - String dbName = nameParts.get(0); - return getTable(ctlName, dbName, nameParts.get(1), getConnectContext().getEnv()); - } - case 3: { // catalog.db.table - return getTable(nameParts.get(0), nameParts.get(1), nameParts.get(2), getConnectContext().getEnv()); - } - default: - throw new IllegalStateException("Table name [" + String.join(".", nameParts) + "] is invalid."); - } - } - - /** - * Find table from catalog. - */ - public TableIf getTable(String ctlName, String dbName, String tableName, Env env) { - CatalogIf catalog = env.getCatalogMgr().getCatalog(ctlName); - if (catalog == null) { - throw new RuntimeException("Catalog [" + ctlName + "] does not exist."); - } - DatabaseIf db = catalog.getDbNullable(dbName); - if (db == null) { - throw new RuntimeException("Database [" + dbName + "] does not exist in catalog [" + ctlName + "]."); - } - - TableIf table = db.getTableNullable(tableName); - if (table == null) { - throw new RuntimeException("Table [" + tableName + "] does not exist in database [" + dbName + "]."); - } - return table; - - } - /** * Used to lock table */ @@ -573,12 +408,12 @@ public static class Lock implements AutoCloseable { */ public Lock(LogicalPlan plan, CascadesContext cascadesContext) { this.cascadesContext = cascadesContext; - // tables can also be load from dump file - if (cascadesContext.getTables() == null || cascadesContext.getTables().isEmpty()) { - cascadesContext.extractTables(plan); - cascadesContext.getStatementContext().setTables(cascadesContext.getTables()); + // if tables loaded from dump file, it is not empty here + if (cascadesContext.getStatementContext().getTables() == null + || cascadesContext.getStatementContext().getTables().isEmpty()) { + cascadesContext.getStatementContext().extractTables(plan); } - for (TableIf table : cascadesContext.tables.values()) { + for (TableIf table : cascadesContext.getStatementContext().getTables().values()) { if (!table.needReadLockWhenPlan()) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 391088abdc827be..0e3019d070bfc28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -239,7 +239,7 @@ protected Plan planWithoutLock( analyze(showAnalyzeProcess(explainLevel, showPlanProcess)); // minidump of input must be serialized first, this process ensure minidump string not null try { - MinidumpUtils.serializeInputsToDumpFile(plan, cascadesContext.getTables()); + MinidumpUtils.serializeInputsToDumpFile(plan, cascadesContext.getStatementContext().getTables()); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 46d87608a577a80..e1873bd460b5247 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -18,12 +18,18 @@ package org.apache.doris.nereids; import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.FormatOptions; import org.apache.doris.common.Id; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundRelation; +import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.hint.Hint; import org.apache.doris.nereids.memo.Group; @@ -35,13 +41,20 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.PlaceholderId; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.TableId; import org.apache.doris.nereids.trees.plans.algebra.Relation; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; @@ -68,6 +81,7 @@ import java.util.BitSet; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -76,6 +90,7 @@ import java.util.Stack; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; /** @@ -211,6 +226,26 @@ public StatementContext(ConnectContext connectContext, OriginStatement originSta } } + public void setTables(Map, TableIf> tables) { + this.tables = tables; + } + + /** + * Extract tables. + */ + public void extractTables(LogicalPlan logicalPlan) { + Set> tableNames = getTables(logicalPlan); + tables = Maps.newHashMap(); + for (List tableName : tableNames) { + try { + TableIf table = getTable(tableName); + tables.put(table.getFullQualifiers(), table); + } catch (Throwable e) { + // IGNORE + } + } + } + public Map, TableIf> getTables() { if (tables == null) { tables = Maps.newHashMap(); @@ -218,8 +253,131 @@ public Map, TableIf> getTables() { return tables; } - public void setTables(Map, TableIf> tables) { - this.tables = tables; + private Set> getTables(LogicalPlan logicalPlan) { + final Set> tableNames = new HashSet<>(); + logicalPlan.foreach(p -> { + if (p instanceof LogicalFilter) { + tableNames.addAll(extractTableNamesFromFilter((LogicalFilter) p)); + } else if (p instanceof LogicalCTE) { + tableNames.addAll(extractTableNamesFromCTE((LogicalCTE) p)); + } else if (p instanceof LogicalProject) { + tableNames.addAll(extractTableNamesFromProject((LogicalProject) p)); + } else if (p instanceof LogicalHaving) { + tableNames.addAll(extractTableNamesFromHaving((LogicalHaving) p)); + } else if (p instanceof UnboundOneRowRelation) { + tableNames.addAll(extractTableNamesFromOneRowRelation((UnboundOneRowRelation) p)); + } else { + Set logicalPlans = p.collect( + n -> (n instanceof UnboundRelation || n instanceof UnboundTableSink)); + for (LogicalPlan plan : logicalPlans) { + if (plan instanceof UnboundRelation) { + tableNames.add(((UnboundRelation) plan).getNameParts()); + } else if (plan instanceof UnboundTableSink) { + tableNames.add(((UnboundTableSink) plan).getNameParts()); + } else { + throw new AnalysisException("get tables from plan failed. meet unknown type node " + plan); + } + } + } + }); + return tableNames; + } + + private Set> extractTableNamesFromHaving(LogicalHaving having) { + Set subqueryExprs = having.getPredicate() + .collect(SubqueryExpr.class::isInstance); + Set> tableNames = new HashSet<>(); + for (SubqueryExpr expr : subqueryExprs) { + LogicalPlan plan = expr.getQueryPlan(); + tableNames.addAll(getTables(plan)); + } + return tableNames; + } + + private Set> extractTableNamesFromOneRowRelation(UnboundOneRowRelation oneRowRelation) { + Set subqueryExprs = oneRowRelation.getProjects().stream() + .>map(p -> p.collect(SubqueryExpr.class::isInstance)) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + Set> tableNames = new HashSet<>(); + for (SubqueryExpr expr : subqueryExprs) { + LogicalPlan plan = expr.getQueryPlan(); + tableNames.addAll(getTables(plan)); + } + return tableNames; + } + + private Set> extractTableNamesFromProject(LogicalProject project) { + Set subqueryExprs = project.getProjects().stream() + .>map(p -> p.collect(SubqueryExpr.class::isInstance)) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + Set> tableNames = new HashSet<>(); + for (SubqueryExpr expr : subqueryExprs) { + LogicalPlan plan = expr.getQueryPlan(); + tableNames.addAll(getTables(plan)); + } + return tableNames; + } + + private Set> extractTableNamesFromFilter(LogicalFilter filter) { + Set subqueryExprs = filter.getPredicate() + .collect(SubqueryExpr.class::isInstance); + Set> tableNames = new HashSet<>(); + for (SubqueryExpr expr : subqueryExprs) { + LogicalPlan plan = expr.getQueryPlan(); + tableNames.addAll(getTables(plan)); + } + return tableNames; + } + + private Set> extractTableNamesFromCTE(LogicalCTE cte) { + List> subQueryAliases = cte.getAliasQueries(); + Set> tableNames = new HashSet<>(); + for (LogicalSubQueryAlias subQueryAlias : subQueryAliases) { + tableNames.addAll(getTables(subQueryAlias)); + } + return tableNames; + } + + private TableIf getTable(List nameParts) { + switch (nameParts.size()) { + case 1: { // table + String ctlName = getConnectContext().getEnv().getCurrentCatalog().getName(); + String dbName = getConnectContext().getDatabase(); + return getTable(ctlName, dbName, nameParts.get(0), getConnectContext().getEnv()); + } + case 2: { // db.table + String ctlName = getConnectContext().getEnv().getCurrentCatalog().getName(); + String dbName = nameParts.get(0); + return getTable(ctlName, dbName, nameParts.get(1), getConnectContext().getEnv()); + } + case 3: { // catalog.db.table + return getTable(nameParts.get(0), nameParts.get(1), nameParts.get(2), getConnectContext().getEnv()); + } + default: + throw new IllegalStateException("Table name [" + String.join(".", nameParts) + "] is invalid."); + } + } + + /** + * Find table from catalog. + */ + public TableIf getTable(String ctlName, String dbName, String tableName, Env env) { + CatalogIf catalog = env.getCatalogMgr().getCatalog(ctlName); + if (catalog == null) { + throw new RuntimeException("Catalog [" + ctlName + "] does not exist."); + } + DatabaseIf db = catalog.getDbNullable(dbName); + if (db == null) { + throw new RuntimeException("Database [" + dbName + "] does not exist in catalog [" + ctlName + "]."); + } + + TableIf table = db.getTableNullable(tableName); + if (table == null) { + throw new RuntimeException("Table [" + tableName + "] does not exist in database [" + dbName + "]."); + } + return table; } /** get table by table name, try to get from information from dumpfile first */