Skip to content

Commit

Permalink
[feat](Nereids) support refresh database command (#44298)
Browse files Browse the repository at this point in the history
Issue Number: close ##42579
  • Loading branch information
vinlee19 authored Dec 2, 2024
1 parent b49374d commit 3e41c91
Show file tree
Hide file tree
Showing 7 changed files with 218 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,7 @@ channelDescription

supportedRefreshStatement
: REFRESH CATALOG name=identifier propertyClause? #refreshCatalog
| REFRESH DATABASE name=multipartIdentifier propertyClause? #refreshDatabase
;

supportedCleanStatement
Expand All @@ -445,7 +446,6 @@ supportedCleanStatement

unsupportedRefreshStatement
: REFRESH TABLE name=multipartIdentifier #refreshTable
| REFRESH DATABASE name=multipartIdentifier propertyClause? #refreshDatabase
| REFRESH LDAP (ALL | (FOR user=identifierOrText)) #refreshLdap
;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@
import org.apache.doris.nereids.DorisParser.RecoverPartitionContext;
import org.apache.doris.nereids.DorisParser.RecoverTableContext;
import org.apache.doris.nereids.DorisParser.RefreshCatalogContext;
import org.apache.doris.nereids.DorisParser.RefreshDatabaseContext;
import org.apache.doris.nereids.DorisParser.RefreshMTMVContext;
import org.apache.doris.nereids.DorisParser.RefreshMethodContext;
import org.apache.doris.nereids.DorisParser.RefreshScheduleContext;
Expand Down Expand Up @@ -618,6 +619,7 @@
import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause;
import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
Expand Down Expand Up @@ -4351,6 +4353,25 @@ public Object visitRefreshCatalog(RefreshCatalogContext ctx) {
}

@Override
public RefreshDatabaseCommand visitRefreshDatabase(RefreshDatabaseContext ctx) {
Map<String, String> properties = visitPropertyClause(ctx.propertyClause()) == null ? Maps.newHashMap()
: visitPropertyClause(ctx.propertyClause());
List<String> parts = visitMultipartIdentifier(ctx.name);
int size = parts.size();
if (size == 0) {
throw new ParseException("database name can't be empty");
}
String dbName = parts.get(size - 1);

// [db].
if (size == 1) {
return new RefreshDatabaseCommand(dbName, properties);
} else if (parts.size() == 2) { // [ctl,db].
return new RefreshDatabaseCommand(parts.get(0), dbName, properties);
}
throw new ParseException("Only one dot can be in the name: " + String.join(".", parts));
}

public LogicalPlan visitShowLastInsert(ShowLastInsertContext ctx) {
return new ShowLastInsertCommand();
}
Expand Down Expand Up @@ -4379,7 +4400,6 @@ public LogicalPlan visitShowPartitionId(ShowPartitionIdContext ctx) {
partitionId = Long.parseLong(ctx.partitionId.getText());
}
return new ShowPartitionIdCommand(partitionId);

}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ public enum PlanType {
SET_USER_PROPERTIES_COMMAND,
SET_DEFAULT_STORAGE_VAULT_COMMAND,
REFRESH_CATALOG_COMMAND,
REFRESH_DATABASE_COMMAND,
PREPARED_COMMAND,
EXECUTE_COMMAND,
DROP_SQL_BLOCK_RULE_COMMAND,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.trees.plans.commands.refresh;

import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.InfoSchemaDb;
import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.ExternalObjectLog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;

import com.google.common.base.Strings;

import java.util.Map;

/**
* Refresh database.
*/
public class RefreshDatabaseCommand extends Command implements ForwardWithSync {
private static final String INVALID_CACHE = "invalid_cache";

private String catalogName;
private String dbName;
private Map<String, String> properties;
private boolean invalidCache = false;

public RefreshDatabaseCommand(String dbName, Map<String, String> properties) {
super(PlanType.REFRESH_DATABASE_COMMAND);
this.dbName = dbName;
this.properties = properties;
}

public RefreshDatabaseCommand(String catalogName, String dbName, Map<String, String> properties) {
super(PlanType.REFRESH_DATABASE_COMMAND);
this.catalogName = catalogName;
this.dbName = dbName;
this.properties = properties;
}

private void validate(ConnectContext ctx) throws AnalysisException {
if (Strings.isNullOrEmpty(catalogName)) {
catalogName = ConnectContext.get().getCurrentCatalog().getName();
}
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName);
}

// Don't allow dropping 'information_schema' database
if (dbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) {

ErrorReport.reportAnalysisException(
ErrorCode.ERR_DBACCESS_DENIED_ERROR, ctx.getQualifiedUser(), dbName);
}
// Don't allow dropping 'mysql' database
if (dbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) {
ErrorReport.reportAnalysisException(
ErrorCode.ERR_DBACCESS_DENIED_ERROR, ctx.getQualifiedUser(), dbName);
}
// check access
if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), catalogName,
dbName, PrivPredicate.SHOW)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED_ERROR,
PrivPredicate.SHOW.getPrivs().toString(), dbName);
}
String invalidConfig = properties == null ? null : properties.get(INVALID_CACHE);
// Default is to invalid cache.
invalidCache = invalidConfig == null || invalidConfig.equalsIgnoreCase("true");
}

/**
* Refresh database
*/
public void handleRefreshDb() throws DdlException {
Env env = Env.getCurrentEnv();
CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
if (catalog == null) {
throw new DdlException("Catalog " + catalogName + " doesn't exist.");
}
if (!(catalog instanceof ExternalCatalog)) {
throw new DdlException("Only support refresh database in external catalog");
}
DatabaseIf db = catalog.getDbOrDdlException(dbName);
((ExternalDatabase<?>) db).setUnInitialized(invalidCache);

ExternalObjectLog log = new ExternalObjectLog();
log.setCatalogId(catalog.getId());
log.setDbId(db.getId());
log.setInvalidCache(invalidCache);
Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log);
}

@Override
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
validate(ctx);
handleRefreshDb();
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitRefreshDatabaseCommand(this, context);
}

/**
* refresh database statement.
*/
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("REFRESH DATABASE ");
if (catalogName != null) {
sb.append("`").append(catalogName).append("`.");
}
sb.append("`").append(dbName).append("`");
return sb.toString();
}

@Override
public StmtType stmtType() {
return StmtType.REFRESH;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@
import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
import org.apache.doris.nereids.trees.plans.commands.load.CreateRoutineLoadCommand;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;

/** CommandVisitor. */
public interface CommandVisitor<R, C> {
Expand Down Expand Up @@ -353,6 +354,10 @@ default R visitShowViewCommand(ShowViewCommand showViewCommand, C context) {
return visitCommand(showViewCommand, context);
}

default R visitRefreshDatabaseCommand(RefreshDatabaseCommand refreshDatabaseCommand, C context) {
return visitCommand(refreshDatabaseCommand, context);
}

default R visitShowBackendsCommand(ShowBackendsCommand showBackendsCommand, C context) {
return visitCommand(showBackendsCommand, context);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
-- !database --
DORIS
Doris
doris
Expand All @@ -19,7 +19,7 @@ show_test_do_not_modify
114 abf
115 abg

-- !sql --
-- !preceding_create_external_database --
DORIS
Doris
doris
Expand All @@ -29,7 +29,7 @@ init_db
mysql
show_test_do_not_modify

-- !sql --
-- !subsequent_create_external_database --
DORIS
Doris
doris
Expand All @@ -40,7 +40,21 @@ mysql
new_mysql_db
show_test_do_not_modify

-- !sql --
-- !sql_show_tables --

-- !preceding_refresh_database --

-- !subsequent_refresh_database --
new_mysql_table1

-- !preceding_refresh_database --
new_mysql_table1

-- !subsequent_refresh_database --
new_mysql_table1
new_mysql_table2

-- !preceding_drop_external_database --
DORIS
Doris
doris
Expand All @@ -51,7 +65,7 @@ mysql
new_mysql_db
show_test_do_not_modify

-- !sql --
-- !subsequent_drop_external_database --
DORIS
Doris
doris
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ suite("test_nereids_refresh_catalog", "p0,external,mysql,external_docker,externa
String mysql_port = context.config.otherConfigs.get("mysql_57_port");
String ex_tb0 = "ex_tb0";
String new_mysql_db = "new_mysql_db";
String new_mysql_table1 = "new_mysql_table1";
String new_mysql_table2 = "new_mysql_table2";

sql """drop catalog if exists ${catalog_name} """

Expand All @@ -43,27 +45,41 @@ suite("test_nereids_refresh_catalog", "p0,external,mysql,external_docker,externa
"driver_class" = "com.mysql.cj.jdbc.Driver"
);"""

sql """switch ${catalog_name}"""
sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists ${new_mysql_db}");"""
sql """switch ${catalog_name}"""

qt_sql """show databases;"""
qt_database """show databases;"""
sql """ use ${ex_db_name}"""

qt_ex_tb0_where """select id from ${ex_tb0} where id = 111;"""
order_qt_ex_tb0 """ select id, name from ${ex_tb0} order by id; """
// create database in mysql
sql """CALL EXECUTE_STMT("${catalog_name}", "create database ${new_mysql_db} ;");"""
qt_sql """show databases;"""
qt_preceding_create_external_database """show databases;"""
checkNereidsExecute("refresh catalog ${catalog_name} ;")
qt_sql """show databases;"""
qt_subsequent_create_external_database """show databases;"""

checkNereidsExecute("refresh catalog ${catalog_name} properties ('invalid_cache'='true');")
sql """use ${new_mysql_db}"""
qt_sql_show_tables """show tables;"""

// create table in mysql external database
sql """CALL EXECUTE_STMT("${catalog_name}", "create table ${new_mysql_db}.${new_mysql_table1} (id int, name varchar(20));");"""

qt_preceding_refresh_database """show tables;"""
checkNereidsExecute("refresh database ${new_mysql_db} ;")
qt_subsequent_refresh_database """show tables;"""

sql """CALL EXECUTE_STMT("${catalog_name}", "create table ${new_mysql_db}.${new_mysql_table2} (id int, name varchar(20));");"""
qt_preceding_refresh_database """show tables;"""
checkNereidsExecute("refresh database ${catalog_name}.${new_mysql_db} ;")
qt_subsequent_refresh_database """show tables;"""

sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists ${new_mysql_db} ;");"""
qt_sql """show databases;"""
qt_preceding_drop_external_database """show databases;"""

checkNereidsExecute("refresh catalog ${catalog_name} properties ('invalid_cache'='true');")
qt_sql """show databases;"""
qt_subsequent_drop_external_database """show databases;"""

sql """ drop catalog if exists ${catalog_name} ;"""
}
Expand Down

0 comments on commit 3e41c91

Please sign in to comment.