From e7c176417042ef636aebfc930b7da939c944f1c9 Mon Sep 17 00:00:00 2001 From: Vallish Date: Sat, 16 Nov 2024 14:24:57 +0000 Subject: [PATCH] [Enhancement] (nereids)implement RecoverCommand in nereids --- .../org/apache/doris/nereids/DorisParser.g4 | 8 +- .../java/org/apache/doris/catalog/Env.java | 9 ++ .../doris/datasource/InternalCatalog.java | 27 +++--- .../nereids/RecoverPartitionCommand.java | 80 ++++++++++++++++ .../nereids/parser/LogicalPlanBuilder.java | 21 +++++ .../plans/commands/RecoverTableCommand.java | 77 +++++++++++++++ .../trees/plans/visitor/CommandVisitor.java | 10 ++ .../ddl/recover/test_recover_all.out | 52 +++++++++++ .../ddl/recover/test_recover_all.groovy | 93 +++++++++++++++++++ 9 files changed, 359 insertions(+), 18 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/RecoverPartitionCommand.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RecoverTableCommand.java create mode 100644 regression-test/data/nereids_p0/ddl/recover/test_recover_all.out create mode 100644 regression-test/suites/nereids_p0/ddl/recover/test_recover_all.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index c92afb1c6eac621..15dd5e4e3fbf5eb 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -56,8 +56,8 @@ statementBase | supportedSetStatement #supportedSetStatementAlias | supportedUnsetStatement #supportedUnsetStatementAlias | supportedShowStatement #supportedShowStatementAlias - | unsupportedStatement #unsupported | supportedRecoverStatement #supportedRecoverStatementAlias + | unsupportedStatement #unsupported ; @@ -74,7 +74,6 @@ unsupportedStatement | unsupportedGrantRevokeStatement | unsupportedAdminStatement | unsupportedTransactionStatement - | unsupportedRecoverStatement | unsupportedCancelStatement | unsupportedJobStatement | unsupportedCleanStatement @@ -459,10 +458,7 @@ unsupportedCancelStatement supportedRecoverStatement : RECOVER DATABASE name=identifier id=INTEGER_VALUE? (AS alias=identifier)? #recoverDatabase - ; - -unsupportedRecoverStatement - :RECOVER TABLE name=multipartIdentifier + | RECOVER TABLE name=multipartIdentifier id=INTEGER_VALUE? (AS alias=identifier)? #recoverTable | RECOVER PARTITION name=identifier id=INTEGER_VALUE? (AS alias=identifier)? FROM tableName=multipartIdentifier #recoverPartition diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 8543e18fd5b8435..ebbe6325ef466e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3277,10 +3277,19 @@ public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException { getInternalCatalog().recoverTable(recoverStmt); } + public void recoverTable(String dbName, String tableName, String newTableName, long tableId) throws DdlException { + getInternalCatalog().recoverTable(dbName, tableName, newTableName, tableId); + } + public void recoverPartition(RecoverPartitionStmt recoverStmt) throws DdlException { getInternalCatalog().recoverPartition(recoverStmt); } + public void recoverPartition(String dbName, String tableName, String partitionName, + String newPartitionName, long partitionId) throws DdlException { + getInternalCatalog().recoverPartition(dbName, tableName, partitionName, newPartitionName, partitionId); + } + public void dropCatalogRecycleBin(IdType idType, long id) throws DdlException { getInternalCatalog().dropCatalogRecycleBin(idType, id); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index ef049fca402f373..ac5cfd59a375883 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -662,11 +662,7 @@ public void recoverDatabase(RecoverDbStmt recoverStmt) throws DdlException { recoverDatabase(recoverStmt.getDbName(), recoverStmt.getDbId(), recoverStmt.getNewDbName()); } - public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException { - String dbName = recoverStmt.getDbName(); - String tableName = recoverStmt.getTableName(); - String newTableName = recoverStmt.getNewTableName(); - + public void recoverTable(String dbName, String tableName, String newTableName, long tableId) throws DdlException { Database db = getDbOrDdlException(dbName); db.writeLockOrDdlException(); try { @@ -679,7 +675,7 @@ public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, newTableName); } } - if (!Env.getCurrentRecycleBin().recoverTable(db, tableName, recoverStmt.getTableId(), newTableName)) { + if (!Env.getCurrentRecycleBin().recoverTable(db, tableName, tableId, newTableName)) { ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TABLE, tableName, dbName); } } finally { @@ -687,16 +683,17 @@ public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException { } } - public void recoverPartition(RecoverPartitionStmt recoverStmt) throws DdlException { - String dbName = recoverStmt.getDbName(); - String tableName = recoverStmt.getTableName(); + public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException { + recoverTable(recoverStmt.getDbName(), recoverStmt.getTableName(), + recoverStmt.getNewTableName(), recoverStmt.getTableId()); + } + public void recoverPartition(String dbName, String tableName, String partitionName, + String newPartitionName, long partitionId) throws DdlException { Database db = getDbOrDdlException(dbName); OlapTable olapTable = db.getOlapTableOrDdlException(tableName); olapTable.writeLockOrDdlException(); try { - String partitionName = recoverStmt.getPartitionName(); - String newPartitionName = recoverStmt.getNewPartitionName(); if (Strings.isNullOrEmpty(newPartitionName)) { if (olapTable.getPartition(partitionName) != null) { throw new DdlException("partition[" + partitionName + "] " @@ -710,12 +707,18 @@ public void recoverPartition(RecoverPartitionStmt recoverStmt) throws DdlExcepti } Env.getCurrentRecycleBin().recoverPartition(db.getId(), olapTable, partitionName, - recoverStmt.getPartitionId(), newPartitionName); + partitionId, newPartitionName); } finally { olapTable.writeUnlock(); } } + public void recoverPartition(RecoverPartitionStmt recoverStmt) throws DdlException { + recoverPartition(recoverStmt.getDbName(), recoverStmt.getTableName(), + recoverStmt.getPartitionName(), recoverStmt.getNewPartitionName(), + recoverStmt.getPartitionId()); + } + public void dropCatalogRecycleBin(IdType idType, long id) throws DdlException { switch (idType) { case DATABASE_ID: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/RecoverPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/RecoverPartitionCommand.java new file mode 100644 index 000000000000000..9cb2f621980eab6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/RecoverPartitionCommand.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * recover partition command + */ +public class RecoverPartitionCommand extends RecoverCommand { + public static final Logger LOG = LogManager.getLogger(RecoverPartitionCommand.class); + private final TableNameInfo dbTblName; + private final String partitionName; + private final long partitionId; + private final String newPartitionName; + + /** + * constructor + */ + public RecoverPartitionCommand(TableNameInfo dbTblName, String partitionName, + long partitionId, String newPartitionName) { + super(PlanType.RECOVER_PARTITION_COMMAND); + this.dbTblName = dbTblName; + this.partitionName = partitionName; + this.partitionId = partitionId; + this.newPartitionName = newPartitionName; + } + + @Override + public void doRun(ConnectContext ctx, StmtExecutor executor) throws UserException { + dbTblName.analyze(ctx); + + // disallow external catalog + Util.prohibitExternalCatalog(dbTblName.getCtl(), this.getClass().getSimpleName()); + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), dbTblName.getCtl(), dbTblName.getDb(), + dbTblName.getTbl(), PrivPredicate.ALTER_CREATE)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "RECOVERY", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbTblName.getDb() + ": " + dbTblName.getTbl()); + } + + Env.getCurrentEnv().recoverPartition(dbTblName.getDb(), dbTblName.getTbl(), + partitionName, newPartitionName, partitionId); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitRecoverPartitionCommand(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 4fe061b5468a71b..35d378cdb60cdf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -163,6 +163,8 @@ import org.apache.doris.nereids.DorisParser.QueryOrganizationContext; import org.apache.doris.nereids.DorisParser.QueryTermContext; import org.apache.doris.nereids.DorisParser.RecoverDatabaseContext; +import org.apache.doris.nereids.DorisParser.RecoverPartitionContext; +import org.apache.doris.nereids.DorisParser.RecoverTableContext; import org.apache.doris.nereids.DorisParser.RefreshMTMVContext; import org.apache.doris.nereids.DorisParser.RefreshMethodContext; import org.apache.doris.nereids.DorisParser.RefreshScheduleContext; @@ -429,6 +431,8 @@ import org.apache.doris.nereids.trees.plans.commands.LoadCommand; import org.apache.doris.nereids.trees.plans.commands.PauseMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.RecoverDatabaseCommand; +import org.apache.doris.nereids.trees.plans.commands.RecoverPartitionCommand; +import org.apache.doris.nereids.trees.plans.commands.RecoverTableCommand; import org.apache.doris.nereids.trees.plans.commands.RefreshMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.ReplayCommand; import org.apache.doris.nereids.trees.plans.commands.ResumeMTMVCommand; @@ -4132,6 +4136,23 @@ public LogicalPlan visitRecoverDatabase(RecoverDatabaseContext ctx) { } @Override + public RecoverTableCommand visitRecoverTable(RecoverTableContext ctx) { + List dbTblNameParts = visitMultipartIdentifier(ctx.name); + String newTableName = (ctx.alias != null) ? ctx.alias.getText() : null; + long tableId = (ctx.id != null) ? Long.parseLong(ctx.id.getText()) : -1; + return new RecoverTableCommand(new TableNameInfo(dbTblNameParts), tableId, newTableName); + } + + @Override + public RecoverPartitionCommand visitRecoverPartition(RecoverPartitionContext ctx) { + String partitionName = ctx.name.getText(); + String newPartitionName = (ctx.alias != null) ? ctx.alias.getText() : null; + long partitionId = (ctx.id != null) ? Long.parseLong(ctx.id.getText()) : -1; + List dbTblNameParts = visitMultipartIdentifier(ctx.tableName); + return new RecoverPartitionCommand(new TableNameInfo(dbTblNameParts), + partitionName, partitionId, newPartitionName); + } + public LogicalPlan visitDropRole(DropRoleContext ctx) { return new DropRoleCommand(ctx.name.getText(), ctx.EXISTS() != null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RecoverTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RecoverTableCommand.java new file mode 100644 index 000000000000000..f28ece422a57fba --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RecoverTableCommand.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * recover table command + */ +public class RecoverTableCommand extends RecoverCommand { + public static final Logger LOG = LogManager.getLogger(RecoverTableCommand.class); + private final TableNameInfo dbTblName; + private final long tableId; + private final String newTableName; + + /** + * constructor + */ + public RecoverTableCommand(TableNameInfo dbTblName, long tableId, String newTableName) { + super(PlanType.RECOVER_TABLE_COMMAND); + this.dbTblName = dbTblName; + this.tableId = tableId; + this.newTableName = newTableName; + } + + @Override + public void doRun(ConnectContext ctx, StmtExecutor executor) throws UserException { + dbTblName.analyze(ctx); + + // disallow external catalog + Util.prohibitExternalCatalog(dbTblName.getCtl(), this.getClass().getSimpleName()); + + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv( + ConnectContext.get(), dbTblName.getCtl(), dbTblName.getDb(), dbTblName.getTbl(), + PrivPredicate.ALTER_CREATE)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "RECOVERY", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbTblName.getDb() + ": " + dbTblName.getTbl()); + } + + Env.getCurrentEnv().recoverTable(dbTblName.getDb(), dbTblName.getTbl(), newTableName, tableId); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitRecoverTableCommand(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 421fdeb4c24d68d..4b052e9ea368312 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -43,6 +43,8 @@ import org.apache.doris.nereids.trees.plans.commands.LoadCommand; import org.apache.doris.nereids.trees.plans.commands.PauseMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.RecoverDatabaseCommand; +import org.apache.doris.nereids.trees.plans.commands.RecoverPartitionCommand; +import org.apache.doris.nereids.trees.plans.commands.RecoverTableCommand; import org.apache.doris.nereids.trees.plans.commands.RefreshMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.ReplayCommand; import org.apache.doris.nereids.trees.plans.commands.ResumeMTMVCommand; @@ -295,6 +297,14 @@ default R visitRecoverDatabaseCommand(RecoverDatabaseCommand recoverDatabaseComm return visitCommand(recoverDatabaseCommand, context); } + default R visitRecoverTableCommand(RecoverTableCommand recoverTableCommand, C context) { + return visitCommand(recoverTableCommand, context); + } + + default R visitRecoverPartitionCommand(RecoverPartitionCommand recoverPartitionCommand, C context) { + return visitCommand(recoverPartitionCommand, context); + } + default R visitDropRoleCommand(DropRoleCommand dropRoleCommand, C context) { return visitCommand(dropRoleCommand, context); } diff --git a/regression-test/data/nereids_p0/ddl/recover/test_recover_all.out b/regression-test/data/nereids_p0/ddl/recover/test_recover_all.out new file mode 100644 index 000000000000000..c2651d467ae08d1 --- /dev/null +++ b/regression-test/data/nereids_p0/ddl/recover/test_recover_all.out @@ -0,0 +1,52 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +table1 +table2 + +-- !select2 -- +table1 + +-- !select3 -- +table1 +table2 + +-- !select4 -- +table1 + +-- !select5 -- +table1 +table3 + +-- !select_check_1 -- +1 a 2022-01-02 +2 a 2023-01-02 +3 a 2024-01-02 + +-- !select_partitions -- +p4 +p5 + +-- !select_partitions_2 -- +p3 +p4 +p5 + +-- !select_check_1 -- +1 a 2022-01-02 +2 a 2023-01-02 +3 a 2024-01-02 + +-- !select_partitions_3 -- +p4 +p5 + +-- !select_partitions_4 -- +p1 +p4 +p5 + +-- !select_check_1 -- +1 a 2022-01-02 +2 a 2023-01-02 +3 a 2024-01-02 + diff --git a/regression-test/suites/nereids_p0/ddl/recover/test_recover_all.groovy b/regression-test/suites/nereids_p0/ddl/recover/test_recover_all.groovy new file mode 100644 index 000000000000000..5bf0bb0cb464f48 --- /dev/null +++ b/regression-test/suites/nereids_p0/ddl/recover/test_recover_all.groovy @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This will test database recover + +suite("test_recover_all") { + def testTable = "test_table" + def db = "test_recover_all_db" + sql "drop database IF EXISTS $db force" + sql "CREATE DATABASE IF NOT EXISTS $db " + sql "use $db " + sql """ + CREATE TABLE IF NOT EXISTS table1 ( + `id` int(11), + `name` varchar(128), + `da` date + ) + engine=olap + duplicate key(id) + partition by range(da)( + PARTITION p3 VALUES LESS THAN ('2023-01-01'), + PARTITION p4 VALUES LESS THAN ('2024-01-01'), + PARTITION p5 VALUES LESS THAN ('2025-01-01') + ) + distributed by hash(id) buckets 2 + properties( + "replication_num"="1", + "light_schema_change"="true" + ); + """ + sql """ + CREATE TABLE IF NOT EXISTS table2 ( + `id` int(11), + `name` varchar(128), + `da` date + ) + engine=olap + duplicate key(id) + partition by range(da)( + PARTITION p3 VALUES LESS THAN ('2023-01-01'), + PARTITION p4 VALUES LESS THAN ('2024-01-01'), + PARTITION p5 VALUES LESS THAN ('2025-01-01') + ) + distributed by hash(id) buckets 2 + properties( + "replication_num"="1", + "light_schema_change"="true" + ); + """ + qt_select "show tables"; + + sql " drop table table2" + qt_select2 "show tables"; + checkNereidsExecute("recover table table2;") + qt_select3 "show tables"; + sql " drop table table2" + qt_select4 "show tables"; + checkNereidsExecute("recover table table2 as table3;") + qt_select5 "show tables"; + + sql """ insert into table3 values(1, 'a', '2022-01-02'); """ + sql """ insert into table3 values(2, 'a', '2023-01-02'); """ + sql """ insert into table3 values(3, 'a', '2024-01-02'); """ + sql """ SYNC;""" + + qt_select_check_1 """ select * from table3 order by id,name,da; """ + sql """ ALTER TABLE table3 DROP PARTITION p3"""; + qt_select_partitions "select PARTITION_NAME from information_schema.partitions where TABLE_NAME = \"table3\""; + sql """ recover partition p3 from table3; """ + qt_select_partitions_2 "select PARTITION_NAME from information_schema.partitions where TABLE_NAME = \"table3\""; + + qt_select_check_1 """ select * from table3 order by id,name,da; """ + sql """ ALTER TABLE table3 DROP PARTITION p3"""; + qt_select_partitions_3 "select PARTITION_NAME from information_schema.partitions where TABLE_NAME = \"table3\""; + sql """ recover partition p3 as p1 from table3; """ + qt_select_partitions_4 "select PARTITION_NAME from information_schema.partitions where TABLE_NAME = \"table3\""; + qt_select_check_1 """ select * from table3 order by id,name,da; """ +} +