Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[feature](nereids)support alter table command #44058

Merged
merged 15 commits into from
Dec 27, 2024
42 changes: 21 additions & 21 deletions fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
Original file line number Diff line number Diff line change
Expand Up @@ -202,15 +202,24 @@ supportedCreateStatement

supportedAlterStatement
: ALTER VIEW name=multipartIdentifier (LEFT_PAREN cols=simpleColumnDefs RIGHT_PAREN)?
AS query #alterView
| ALTER STORAGE VAULT name=multipartIdentifier properties=propertyClause #alterStorageVault
| ALTER ROLE role=identifier commentSpec #alterRole
AS query #alterView
| ALTER STORAGE VAULT name=multipartIdentifier properties=propertyClause #alterStorageVault
| ALTER ROLE role=identifier commentSpec #alterRole
| ALTER WORKLOAD GROUP name=identifierOrText
properties=propertyClause? #alterWorkloadGroup
properties=propertyClause? #alterWorkloadGroup
| ALTER WORKLOAD POLICY name=identifierOrText
properties=propertyClause? #alterWorkloadPolicy
| ALTER SQL_BLOCK_RULE name=identifier properties=propertyClause? #alterSqlBlockRule
| ALTER CATALOG name=identifier MODIFY COMMENT comment=STRING_LITERAL #alterCatalogComment
properties=propertyClause? #alterWorkloadPolicy
| ALTER SQL_BLOCK_RULE name=identifier properties=propertyClause? #alterSqlBlockRule
| ALTER CATALOG name=identifier MODIFY COMMENT comment=STRING_LITERAL #alterCatalogComment
| ALTER ROLE role=identifier commentSpec #alterRole
| ALTER TABLE tableName=multipartIdentifier
alterTableClause (COMMA alterTableClause)* #alterTable
| ALTER TABLE tableName=multipartIdentifier ADD ROLLUP
addRollupClause (COMMA addRollupClause)* #alterTableAddRollup
| ALTER TABLE tableName=multipartIdentifier DROP ROLLUP
dropRollupClause (COMMA dropRollupClause)* #alterTableDropRollup
| ALTER TABLE name=multipartIdentifier
SET LEFT_PAREN propertyItemList RIGHT_PAREN #alterTableProperties
;

supportedDropStatement
Expand Down Expand Up @@ -573,13 +582,7 @@ privilegeList
;

unsupportedAlterStatement
: ALTER TABLE tableName=multipartIdentifier
alterTableClause (COMMA alterTableClause)* #alterTable
| ALTER TABLE tableName=multipartIdentifier ADD ROLLUP
addRollupClause (COMMA addRollupClause)* #alterTableAddRollup
| ALTER TABLE tableName=multipartIdentifier DROP ROLLUP
dropRollupClause (COMMA dropRollupClause)* #alterTableDropRollup
| ALTER SYSTEM alterSystemClause #alterSystem
: ALTER SYSTEM alterSystemClause #alterSystem
| ALTER DATABASE name=identifier SET (DATA |REPLICA | TRANSACTION)
QUOTA INTEGER_VALUE identifier? #alterDatabaseSetQuota
| ALTER DATABASE name=identifier RENAME newName=identifier #alterDatabaseRename
Expand All @@ -593,8 +596,6 @@ unsupportedAlterStatement
SET LEFT_PAREN propertyItemList RIGHT_PAREN #alterColocateGroup
| ALTER ROUTINE LOAD FOR name=multipartIdentifier properties=propertyClause?
(FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)? #alterRoutineLoad
| ALTER TABLE name=multipartIdentifier
SET LEFT_PAREN propertyItemList RIGHT_PAREN #alterTableProperties
| ALTER STORAGE POLICY name=identifierOrText
properties=propertyClause #alterStoragePlicy
| ALTER USER (IF EXISTS)? grantUserIdentify
Expand Down Expand Up @@ -638,20 +639,19 @@ addRollupClause

alterTableClause
: ADD COLUMN columnDef columnPosition? toRollup? properties=propertyClause? #addColumnClause
| ADD COLUMN LEFT_PAREN columnDef (COMMA columnDef)* RIGHT_PAREN
| ADD COLUMN LEFT_PAREN columnDefs RIGHT_PAREN
toRollup? properties=propertyClause? #addColumnsClause
| DROP COLUMN name=identifier fromRollup? properties=propertyClause? #dropColumnClause
| MODIFY COLUMN columnDef columnPosition? fromRollup?
properties=propertyClause? #modifyColumnClause
| ORDER BY identifierList fromRollup? properties=propertyClause? #reorderColumnsClause
| ADD TEMPORARY? (lessThanPartitionDef | fixedPartitionDef | inPartitionDef)
(LEFT_PAREN partitionProperties=propertyItemList RIGHT_PAREN)?
| ADD TEMPORARY? partitionDef
(DISTRIBUTED BY (HASH hashKeys=identifierList | RANDOM)
(BUCKETS (INTEGER_VALUE | autoBucket=AUTO))?)?
properties=propertyClause? #addPartitionClause
| DROP TEMPORARY? PARTITION (IF EXISTS)? partitionName=identifier FORCE?
(FROM INDEX indexName=identifier)? #dropPartitionClause
| MODIFY TEMPORARY? PARTITION (IF EXISTS)?
| MODIFY TEMPORARY? PARTITION
(partitionName=identifier | partitionNames=identifierList
| LEFT_PAREN ASTERISK RIGHT_PAREN)
SET LEFT_PAREN partitionProperties=propertyItemList RIGHT_PAREN #modifyPartitionClause
Expand Down Expand Up @@ -1379,7 +1379,7 @@ indexDefs
;

indexDef
: INDEX (IF NOT EXISTS)? indexName=identifier cols=identifierList (USING indexType=(BITMAP | INVERTED | NGRAM_BF))? (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)? (COMMENT comment=STRING_LITERAL)?
: INDEX (ifNotExists=IF NOT EXISTS)? indexName=identifier cols=identifierList (USING indexType=(BITMAP | INVERTED | NGRAM_BF))? (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)? (COMMENT comment=STRING_LITERAL)?
;

partitionsDef
Expand Down
130 changes: 126 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.PropertyAnalyzer.RewriteProperty;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
import org.apache.doris.persist.AlterMTMV;
import org.apache.doris.persist.AlterViewInfo;
Expand All @@ -91,6 +92,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
Expand Down Expand Up @@ -147,8 +149,31 @@ private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable,
}
stmt.rewriteAlterClause(olapTable);

// check conflict alter ops first
alterClauses.addAll(stmt.getOps());
return processAlterOlapTableInternal(alterClauses, olapTable, db, stmt.toSql());
}

private boolean processAlterOlapTable(AlterTableCommand command, OlapTable olapTable,
List<AlterClause> alterClauses,
Database db) throws UserException {
if (olapTable.getDataSortInfo() != null
&& olapTable.getDataSortInfo().getSortType() == TSortType.ZORDER) {
throw new UserException("z-order table can not support schema change!");
}

// check conflict alter ops first
alterClauses.addAll(command.getOps());
return processAlterOlapTableInternal(alterClauses, olapTable, db, command.toSql());
}

private boolean processAlterOlapTableInternal(List<AlterClause> alterClauses, OlapTable olapTable,
Database db, String sql) throws UserException {
if (olapTable.getDataSortInfo() != null
&& olapTable.getDataSortInfo().getSortType() == TSortType.ZORDER) {
throw new UserException("z-order table can not support schema change!");
}

// check conflict alter ops first
AlterOperations currentAlterOps = new AlterOperations();
currentAlterOps.checkConflict(alterClauses);

Expand Down Expand Up @@ -226,7 +251,7 @@ private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable,
((SchemaChangeHandler) schemaChangeHandler).updateBinlogConfig(db, olapTable, alterClauses);
} else if (currentAlterOps.hasSchemaChangeOp()) {
// if modify storage type to v2, do schema change to convert all related tablets to segment v2 format
schemaChangeHandler.process(stmt.toSql(), alterClauses, db, olapTable);
schemaChangeHandler.process(sql, alterClauses, db, olapTable);
} else if (currentAlterOps.hasRollupOp()) {
materializedViewHandler.process(alterClauses, db, olapTable);
} else if (currentAlterOps.hasPartitionOp()) {
Expand Down Expand Up @@ -407,10 +432,22 @@ public void replayModifyComment(ModifyCommentOperationLog operation) throws Meta
}
}

private void processAlterExternalTable(AlterTableStmt stmt, Table externalTable, Database db) throws UserException {
private void processAlterExternalTable(AlterTableStmt stmt, Table externalTable, Database db)
throws UserException {
stmt.checkExternalTableOperationAllow(externalTable);
processAlterExternalTableInternal(stmt.getOps(), externalTable, db);
}

private void processAlterExternalTable(AlterTableCommand command, Table externalTable, Database db)
throws UserException {
List<AlterClause> alterClauses = new ArrayList<>();
alterClauses.addAll(command.getOps());
processAlterExternalTableInternal(alterClauses, externalTable, db);
}

private void processAlterExternalTableInternal(List<AlterClause> alterClauses, Table externalTable, Database db)
throws UserException {
// check conflict alter ops first
List<AlterClause> alterClauses = stmt.getOps();
AlterOperations currentAlterOps = new AlterOperations();
currentAlterOps.checkConflict(alterClauses);
if (currentAlterOps.hasRenameOp()) {
Expand Down Expand Up @@ -572,6 +609,91 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException {
}
}

public void processAlterTable(AlterTableCommand command) throws UserException {
TableNameInfo dbTableName = command.getTbl();
String ctlName = dbTableName.getCtl();
String dbName = dbTableName.getDb();
String tableName = dbTableName.getTbl();
DatabaseIf dbIf = Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrException(ctlName, catalog -> new DdlException("Unknown catalog " + catalog))
.getDbOrDdlException(dbName);
TableIf tableIf = dbIf.getTableOrDdlException(tableName);
List<AlterClause> alterClauses = Lists.newArrayList();
// some operations will take long time to process, need to be done outside the table lock
boolean needProcessOutsideTableLock = false;
switch (tableIf.getType()) {
case MATERIALIZED_VIEW:
case OLAP:
OlapTable olapTable = (OlapTable) tableIf;
needProcessOutsideTableLock = processAlterOlapTable(command, olapTable, alterClauses, (Database) dbIf);
break;
case ODBC:
case JDBC:
case HIVE:
case MYSQL:
case ELASTICSEARCH:
processAlterExternalTable(command, (Table) tableIf, (Database) dbIf);
return;
case HMS_EXTERNAL_TABLE:
case JDBC_EXTERNAL_TABLE:
case ICEBERG_EXTERNAL_TABLE:
case PAIMON_EXTERNAL_TABLE:
case MAX_COMPUTE_EXTERNAL_TABLE:
case HUDI_EXTERNAL_TABLE:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
alterClauses.addAll(command.getOps());
setExternalTableAutoAnalyzePolicy((ExternalTable) tableIf, alterClauses);
return;
default:
throw new DdlException("Do not support alter "
+ tableIf.getType().toString() + " table[" + tableName + "]");
}

Database db = (Database) dbIf;
// the following ops should done outside table lock. because it contain synchronized create operation
if (needProcessOutsideTableLock) {
Preconditions.checkState(alterClauses.size() == 1);
AlterClause alterClause = alterClauses.get(0);
if (alterClause instanceof AddPartitionClause) {
if (!((AddPartitionClause) alterClause).isTempPartition()) {
DynamicPartitionUtil.checkAlterAllowed(
(OlapTable) db.getTableOrMetaException(tableName, TableType.OLAP));
}
Env.getCurrentEnv().addPartition(db, tableName, (AddPartitionClause) alterClause, false, 0, true);
} else if (alterClause instanceof AddPartitionLikeClause) {
if (!((AddPartitionLikeClause) alterClause).getIsTempPartition()) {
DynamicPartitionUtil.checkAlterAllowed(
(OlapTable) db.getTableOrMetaException(tableName, TableType.OLAP));
}
Env.getCurrentEnv().addPartitionLike(db, tableName, (AddPartitionLikeClause) alterClause);
} else if (alterClause instanceof ModifyPartitionClause) {
ModifyPartitionClause clause = ((ModifyPartitionClause) alterClause);
Map<String, String> properties = clause.getProperties();
List<String> partitionNames = clause.getPartitionNames();
((SchemaChangeHandler) schemaChangeHandler).updatePartitionsProperties(
db, tableName, partitionNames, properties);
OlapTable olapTable = (OlapTable) tableIf;
olapTable.writeLockOrDdlException();
try {
modifyPartitionsProperty(db, olapTable, partitionNames, properties, clause.isTempPartition());
} finally {
olapTable.writeUnlock();
}
} else if (alterClause instanceof ModifyTablePropertiesClause) {
Map<String, String> properties = alterClause.getProperties();
((SchemaChangeHandler) schemaChangeHandler).updateTableProperties(db, tableName, properties);
} else if (alterClause instanceof AlterMultiPartitionClause) {
if (!((AlterMultiPartitionClause) alterClause).isTempPartition()) {
DynamicPartitionUtil.checkAlterAllowed(
(OlapTable) db.getTableOrMetaException(tableName, TableType.OLAP));
}
Env.getCurrentEnv().addMultiPartitions(db, tableName, (AlterMultiPartitionClause) alterClause);
} else {
throw new DdlException("Invalid alter operation: " + alterClause.getOpType());
}
}
}

// entry of processing replace table
private void processReplaceTable(Database db, OlapTable origTable, List<AlterClause> alterClauses)
throws UserException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -572,7 +572,7 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol
throw new DdlException("Aggregate method must be specified for value column: " + modColumn.getName());
}
} else if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
if (null != modColumn.getAggregationType()) {
if (null != modColumn.getAggregationType() && modColumn.isKey()) {
throw new DdlException("Can not assign aggregation method" + " on column in Unique data model table: "
+ modColumn.getName());
}
Expand All @@ -584,7 +584,7 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol
}
}
} else {
if (null != modColumn.getAggregationType()) {
if (null != modColumn.getAggregationType() && modColumn.isKey()) {
throw new DdlException(
"Can not assign aggregation method" + " on column in Duplicate data model table: "
+ modColumn.getName());
Expand Down Expand Up @@ -970,7 +970,7 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
}
}
} else if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
if (newColumn.getAggregationType() != null && newColumn.getAggregationType() != AggregateType.NONE) {
if (newColumn.getAggregationType() != null && newColumn.isKey()) {
throw new DdlException(
"Can not assign aggregation method" + " on column in Unique data model table: " + newColName);
}
Expand All @@ -982,7 +982,7 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
}
}
} else {
if (newColumn.getAggregationType() != null) {
if (newColumn.getAggregationType() != null && newColumn.isKey()) {
throw new DdlException(
"Can not assign aggregation method" + " on column in Duplicate data model table: "
+ newColName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
public class AddColumnClause extends AlterTableClause {
private static final Logger LOG = LogManager.getLogger(AddColumnClause.class);
private ColumnDef columnDef;
private String sql;
// Column position
private ColumnPosition colPos;
// if rollupName is null, add to column to base index.
Expand Down Expand Up @@ -68,6 +69,17 @@ public AddColumnClause(ColumnDef columnDef, ColumnPosition colPos, String rollup
this.properties = properties;
}

// for nereids
public AddColumnClause(String sql, Column column, ColumnPosition colPos, String rollupName,
Map<String, String> properties) {
super(AlterOpType.SCHEMA_CHANGE);
this.sql = sql;
this.column = column;
this.colPos = colPos;
this.rollupName = rollupName;
this.properties = properties;
}

@Override
public void analyze(Analyzer analyzer) throws AnalysisException, DdlException {
if (columnDef == null) {
Expand Down Expand Up @@ -122,15 +134,19 @@ public boolean needChangeMTMVState() {

@Override
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("ADD COLUMN ").append(columnDef.toSql());
if (colPos != null) {
sb.append(" ").append(colPos.toSql());
}
if (rollupName != null) {
sb.append(" IN `").append(rollupName).append("`");
if (sql != null) {
return sql;
} else {
StringBuilder sb = new StringBuilder();
sb.append("ADD COLUMN ").append(columnDef.toSql());
if (colPos != null) {
sb.append(" ").append(colPos.toSql());
}
if (rollupName != null) {
sb.append(" IN `").append(rollupName).append("`");
}
return sb.toString();
}
return sb.toString();
}

@Override
Expand Down
Loading
Loading