From a85d631584c2aaf4a7ed2d4313e0ab437b40025c Mon Sep 17 00:00:00 2001 From: Vallish Date: Thu, 21 Nov 2024 08:17:45 +0000 Subject: [PATCH] [Enhancement] (nereids)implement showReplicaDistributionCommand in nereids --- .../org/apache/doris/nereids/DorisParser.g4 | 8 +- .../apache/doris/catalog/MetadataViewer.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 93 ++++++++++++++++ .../doris/nereids/trees/plans/PlanType.java | 1 + .../ShowReplicaDistributionCommand.java | 104 ++++++++++++++++++ .../plans/commands/info/TableRefInfo.java | 75 +++++++++++++ .../trees/plans/visitor/CommandVisitor.java | 6 + 7 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 7e727a7f0419617..9258c9630adfa3c 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -59,6 +59,7 @@ statementBase | supportedRefreshStatement #supportedRefreshStatementAlias | supportedShowStatement #supportedShowStatementAlias | supportedRecoverStatement #supportedRecoverStatementAlias + | supportedAdminStatement #supportedAdminStatementAlias | unsupportedStatement #unsupported ; @@ -226,6 +227,7 @@ supportedShowStatement | SHOW CREATE MATERIALIZED VIEW mvName=identifier ON tableName=multipartIdentifier #showCreateMaterializedView | SHOW BACKENDS #showBackends + | SHOW REPLICA DISTRIBUTION FROM baseTableRef #showReplicaDistribution | SHOW FULL? TRIGGERS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTriggers | SHOW FRONTENDS name=identifier? #showFrontends | SHOW TABLE tableId=INTEGER_VALUE #showTableId @@ -338,7 +340,6 @@ unsupportedShowStatement | SHOW (CLUSTERS | (COMPUTE GROUPS)) #showClusters | SHOW CONVERT_LSC ((FROM | IN) database=multipartIdentifier)? #showConvertLsc | SHOW REPLICA STATUS FROM baseTableRef wildWhere? #showReplicaStatus - | SHOW REPLICA DISTRIBUTION FROM baseTableRef #showREplicaDistribution | SHOW TABLET STORAGE FORMAT VERBOSE? #showTabletStorageFormat | SHOW TABLET DIAGNOSIS tabletId=INTEGER_VALUE #showDiagnoseTablet | SHOW COPY ((FROM | IN) database=multipartIdentifier)? @@ -460,6 +461,10 @@ unsupportedCancelStatement | CANCEL WARM UP JOB wildWhere? #cancelWarmUp ; +supportedAdminStatement + : ADMIN SHOW REPLICA DISTRIBUTION FROM baseTableRef #adminShowReplicaDistribution + ; + supportedRecoverStatement : RECOVER DATABASE name=identifier id=INTEGER_VALUE? (AS alias=identifier)? #recoverDatabase | RECOVER TABLE name=multipartIdentifier @@ -470,7 +475,6 @@ supportedRecoverStatement unsupportedAdminStatement : ADMIN SHOW REPLICA STATUS FROM baseTableRef wildWhere? #adminShowReplicaStatus - | ADMIN SHOW REPLICA DISTRIBUTION FROM baseTableRef #adminShowReplicaDistribution | ADMIN SET REPLICA STATUS PROPERTIES LEFT_PAREN propertyItemList RIGHT_PAREN #adminSetReplicaStatus | ADMIN SET REPLICA VERSION PROPERTIES LEFT_PAREN propertyItemList RIGHT_PAREN #adminSetReplicaVersion | ADMIN REPAIR TABLE baseTableRef #adminRepairTable diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java index fd4a530f4369e1e..4da77267857f673 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java @@ -167,7 +167,7 @@ public static List> getTabletDistribution(ShowReplicaDistributionSt return getTabletDistribution(stmt.getDbName(), stmt.getTblName(), stmt.getPartitionNames()); } - private static List> getTabletDistribution( + public static List> getTabletDistribution( String dbName, String tblName, PartitionNames partitionNames) throws DdlException { DecimalFormat df = new DecimalFormat("00.00 %"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index cd9279c90230486..7d30b3c5ebc54c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.ColumnNullableType; +import org.apache.doris.analysis.PartitionNames; import org.apache.doris.analysis.PassVar; import org.apache.doris.analysis.SetType; import org.apache.doris.analysis.StorageBackend; @@ -48,6 +49,7 @@ import org.apache.doris.mtmv.MTMVRefreshTriggerInfo; import org.apache.doris.nereids.DorisParser; import org.apache.doris.nereids.DorisParser.AddConstraintContext; +import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext; import org.apache.doris.nereids.DorisParser.AggClauseContext; import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext; import org.apache.doris.nereids.DorisParser.AliasQueryContext; @@ -61,6 +63,7 @@ import org.apache.doris.nereids.DorisParser.ArrayLiteralContext; import org.apache.doris.nereids.DorisParser.ArrayRangeContext; import org.apache.doris.nereids.DorisParser.ArraySliceContext; +import org.apache.doris.nereids.DorisParser.BaseTableRefContext; import org.apache.doris.nereids.DorisParser.BitOperationContext; import org.apache.doris.nereids.DorisParser.BooleanExpressionContext; import org.apache.doris.nereids.DorisParser.BooleanLiteralContext; @@ -144,6 +147,7 @@ import org.apache.doris.nereids.DorisParser.NamedExpressionContext; import org.apache.doris.nereids.DorisParser.NamedExpressionSeqContext; import org.apache.doris.nereids.DorisParser.NullLiteralContext; +import org.apache.doris.nereids.DorisParser.OptScanParamsContext; import org.apache.doris.nereids.DorisParser.OutFileClauseContext; import org.apache.doris.nereids.DorisParser.ParenthesizedExpressionContext; import org.apache.doris.nereids.DorisParser.PartitionSpecContext; @@ -175,6 +179,7 @@ import org.apache.doris.nereids.DorisParser.RefreshTriggerContext; import org.apache.doris.nereids.DorisParser.RegularQuerySpecificationContext; import org.apache.doris.nereids.DorisParser.RelationContext; +import org.apache.doris.nereids.DorisParser.RelationHintContext; import org.apache.doris.nereids.DorisParser.ReplaceContext; import org.apache.doris.nereids.DorisParser.ResumeMTMVContext; import org.apache.doris.nereids.DorisParser.RollupDefContext; @@ -221,6 +226,7 @@ import org.apache.doris.nereids.DorisParser.ShowPrivilegesContext; import org.apache.doris.nereids.DorisParser.ShowProcContext; import org.apache.doris.nereids.DorisParser.ShowProcedureStatusContext; +import org.apache.doris.nereids.DorisParser.ShowReplicaDistributionContext; import org.apache.doris.nereids.DorisParser.ShowRepositoriesContext; import org.apache.doris.nereids.DorisParser.ShowRolesContext; import org.apache.doris.nereids.DorisParser.ShowSqlBlockRuleContext; @@ -237,6 +243,7 @@ import org.apache.doris.nereids.DorisParser.SingleStatementContext; import org.apache.doris.nereids.DorisParser.SortClauseContext; import org.apache.doris.nereids.DorisParser.SortItemContext; +import org.apache.doris.nereids.DorisParser.SpecifiedPartitionContext; import org.apache.doris.nereids.DorisParser.StarContext; import org.apache.doris.nereids.DorisParser.StatementDefaultContext; import org.apache.doris.nereids.DorisParser.StepPartitionDefContext; @@ -248,7 +255,9 @@ import org.apache.doris.nereids.DorisParser.SystemVariableContext; import org.apache.doris.nereids.DorisParser.TableAliasContext; import org.apache.doris.nereids.DorisParser.TableNameContext; +import org.apache.doris.nereids.DorisParser.TableSnapshotContext; import org.apache.doris.nereids.DorisParser.TableValuedFunctionContext; +import org.apache.doris.nereids.DorisParser.TabletListContext; import org.apache.doris.nereids.DorisParser.TimestampaddContext; import org.apache.doris.nereids.DorisParser.TimestampdiffContext; import org.apache.doris.nereids.DorisParser.TypeConstructorContext; @@ -488,6 +497,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowReplicaDistributionCommand; import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand; @@ -548,6 +558,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.SimpleColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.info.StepPartition; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo; import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand; @@ -4259,6 +4270,88 @@ public LogicalPlan visitShowProc(ShowProcContext ctx) { return new ShowProcCommand(path); } + private TableScanParams visitTableScanParams(OptScanParamsContext ctx) { + if (ctx != null) { + Map map = visitPropertyItemList(ctx.properties); + return new TableScanParams(ctx.funcName.getText(), map); + } + return null; + } + + private TableSnapshot visitTableSnapshotContext(TableSnapshotContext ctx) { + if (ctx != null) { + if (ctx.TIME() != null) { + return new TableSnapshot(stripQuotes(ctx.time.getText())); + } else { + return new TableSnapshot(Long.parseLong(ctx.version.getText())); + } + } + return null; + } + + private List visitRelationHint(RelationHintContext ctx) { + final List relationHints; + if (ctx != null) { + relationHints = typedVisit(ctx); + } else { + relationHints = ImmutableList.of(); + } + return relationHints; + } + + private PartitionNames visitPartitionNames(SpecifiedPartitionContext ctx) { + if (ctx != null) { + List partitions = new ArrayList<>(); + boolean isTempPart = ctx.TEMPORARY() != null; + if (ctx.identifier() != null) { + partitions.add(ctx.identifier().getText()); + } else { + partitions.addAll(visitIdentifierList(ctx.identifierList())); + } + return new PartitionNames(isTempPart, partitions); + } + return null; + } + + private List visitTabletIdList(TabletListContext ctx) { + List tabletIdList = new ArrayList<>(); + if (ctx != null && ctx.tabletIdList != null) { + ctx.tabletIdList.stream().forEach(tabletToken -> { + tabletIdList.add(Long.parseLong(tabletToken.getText())); + }); + } + return tabletIdList; + } + + private TableRefInfo visitBaseTableRefContext(BaseTableRefContext ctx) { + List nameParts = visitMultipartIdentifier(ctx.multipartIdentifier()); + TableScanParams scanParams = visitTableScanParams(ctx.optScanParams()); + TableSnapshot tableSnapShot = visitTableSnapshotContext(ctx.tableSnapshot()); + PartitionNames partitionNames = visitPartitionNames(ctx.specifiedPartition()); + List tabletIdList = visitTabletIdList(ctx.tabletList()); + + String tableAlias = null; + if (ctx.tableAlias().strictIdentifier() != null) { + tableAlias = ctx.tableAlias().getText(); + } + TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); + List hints = visitRelationHint(ctx.relationHint()); + return new TableRefInfo(new TableNameInfo(nameParts), scanParams, tableSnapShot, partitionNames, + tabletIdList, tableAlias, tableSample, hints); + } + + @Override + public LogicalPlan visitShowReplicaDistribution(ShowReplicaDistributionContext ctx) { + TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef()); + return new ShowReplicaDistributionCommand(tableRefInfo); + } + + @Override + public LogicalPlan visitAdminShowReplicaDistribution(AdminShowReplicaDistributionContext ctx) { + TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef()); + return new ShowReplicaDistributionCommand(tableRefInfo); + } + @Override public LogicalPlan visitShowCreateCatalog(ShowCreateCatalogContext ctx) { return new ShowCreateCatalogCommand(ctx.identifier().getText()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index b4a9aaa51f12218..dbb76481e069ee5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -200,6 +200,7 @@ public enum PlanType { SHOW_PROC_COMMAND, SHOW_PLUGINS_COMMAND, SHOW_PRIVILEGES_COMMAND, + SHOW_REPLICA_DISTRIBUTION_COMMAND, SHOW_REPOSITORIES_COMMAND, SHOW_ROLE_COMMAND, SHOW_STORAGE_ENGINES_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java new file mode 100644 index 000000000000000..21511af5ea911d5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.RedirectStatus; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MetadataViewer; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * show replica distribution command + */ +public class ShowReplicaDistributionCommand extends ShowCommand { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("BackendId").add("ReplicaNum").add("ReplicaSize") + .add("NumGraph").add("NumPercent") + .add("SizeGraph").add("SizePercent") + .add("CloudClusterName").add("CloudClusterId") + .build(); + private final TableRefInfo tableRefInfo; + + /** + * constructor + */ + + public ShowReplicaDistributionCommand(TableRefInfo tableRefInfo) { + super(PlanType.SHOW_REPLICA_DISTRIBUTION_COMMAND); + this.tableRefInfo = tableRefInfo; + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + // check auth + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + tableRefInfo.analyze(ctx); + Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName()); + + List> results; + try { + results = MetadataViewer.getTabletDistribution(tableRefInfo.getTableNameInfo().getDb(), + tableRefInfo.getTableNameInfo().getTbl(), tableRefInfo.getPartitionNames()); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage()); + } + return new ShowResultSet(getMetaData(), results); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitShowReplicaDistributionCommand(this, context); + } + + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String title : TITLE_NAMES) { + builder.addColumn(new Column(title, ScalarType.createVarchar(30))); + } + return builder.build(); + } + + @Override + public RedirectStatus toRedirectStatus() { + if (ConnectContext.get().getSessionVariable().getForwardToMaster()) { + return RedirectStatus.FORWARD_NO_SYNC; + } else { + return RedirectStatus.NO_FORWARD; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java new file mode 100644 index 000000000000000..30914dc6aafa027 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is modified copy of TableRef + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.PartitionNames; +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.analysis.TableSnapshot; +import org.apache.doris.nereids.trees.TableSample; +import org.apache.doris.qe.ConnectContext; + +import java.util.List; + +/** + * table ref info + * baseTableRef + : multipartIdentifier optScanParams? tableSnapshot? specifiedPartition? + tabletList? tableAlias sample? relationHint? + currently tableNameInfo(multipartIdentifier) and specifiedPartition API exposed. + if you need write one and use. + */ +public class TableRefInfo { + private final TableNameInfo tableNameInfo; + private final TableScanParams scanParams; + private final TableSnapshot tableSnapShot; + private final PartitionNames partitionNames; + private final List tabletIdList; + private final String tableAlias; + private final TableSample tableSample; + private final List relationHints; + + /** + * constructor + */ + public TableRefInfo(TableNameInfo tableNameInfo, TableScanParams scanParams, + TableSnapshot tableSnapShot, PartitionNames partitionNames, + List tabletIdList, String tableAlias, + TableSample tableSample, List relationHints) { + this.tableNameInfo = tableNameInfo; + this.scanParams = scanParams; + this.tableSnapShot = tableSnapShot; + this.partitionNames = partitionNames; + this.tabletIdList = tabletIdList; + this.tableAlias = tableAlias; + this.tableSample = tableSample; + this.relationHints = relationHints; + } + + public TableNameInfo getTableNameInfo() { + return tableNameInfo; + } + + public PartitionNames getPartitionNames() { + return partitionNames; + } + + public void analyze(ConnectContext ctx) { + tableNameInfo.analyze(ctx); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 0a7446075fa68cf..cca1f1761ee5785 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -80,6 +80,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowReplicaDistributionCommand; import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand; @@ -414,6 +415,11 @@ default R visitDropUserCommand(DropUserCommand dropUserCommand, C context) { return visitCommand(dropUserCommand, context); } + default R visitShowReplicaDistributionCommand(ShowReplicaDistributionCommand showReplicaDistributedCommand, + C context) { + return visitCommand(showReplicaDistributedCommand, context); + } + default R visitShowTableIdCommand(ShowTableIdCommand showTableIdCommand, C context) { return visitCommand(showTableIdCommand, context); }