[opt](flexible partial update) Don't store table property `enable_uni…

…que_key_skip_bitmap_column` (#46273)

Problem Summary:

When creating table, table property
`enable_unique_key_skip_bitmap_column` indicate whether or not to add
the hidden skip bitmap column for the targeted table. After the table is
created, this property indicate that whether the table has the hidden
skip bitmap column. So we don't need to actually store this property in
table's property map and don't need to pay for keeping consistency
between these two things.

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java

@@ -1385,26 +1385,6 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map<Long,
             }
         }
 
-        // property enable_unique_key_skip_bitmap
-        boolean enableUniqueKeySkipBitmap = false;
-        boolean hasEnableUniqueKeySkipBitmapChanged = false;
-        if (propertyMap.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) {
-            if (!(olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite())) {
-                throw new DdlException("Only allow to alter table property enable_unique_key_skip_bitmap"
-                        + "on unique merge-on-write table.");
-            }
-            try {
-                enableUniqueKeySkipBitmap = PropertyAnalyzer.analyzeUniqueKeySkipBitmapColumn(propertyMap);
-            } catch (AnalysisException e) {
-                throw new DdlException(e.getMessage());
-            }
-            if (enableUniqueKeySkipBitmap) {
-                if (olapTable.getEnableUniqueKeySkipBitmap() != enableUniqueKeySkipBitmap) {
-                    hasEnableUniqueKeySkipBitmapChanged = true;
-                }
-            }
-        }
-
         // begin checking each table
         // ATTN: DO NOT change any meta in this loop
         long tableId = olapTable.getId();
@@ -1480,8 +1460,6 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map<Long,
                 if (olapTable.getStorageFormat() != TStorageFormat.V2) {
                     needAlter = true;
                 }
-            } else if (hasEnableUniqueKeySkipBitmapChanged) {
-                needAlter = true;
             }
 
             if (!needAlter) {
@@ -1599,8 +1577,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map<Long,
             }
         } // end for indices
 
-        if (changedIndexIdToSchema.isEmpty() && !hasIndexChange && !hasRowStoreChanged
-                && !hasEnableUniqueKeySkipBitmapChanged) {
+        if (changedIndexIdToSchema.isEmpty() && !hasIndexChange && !hasRowStoreChanged) {
             throw new DdlException("Nothing is changed. please check your alter stmt.");
         }
 
@@ -1614,8 +1591,6 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map<Long,
         schemaChangeJob.setBloomFilterInfo(hasBfChange, bfColumns, bfFpp);
         schemaChangeJob.setAlterIndexInfo(hasIndexChange, indexes);
         schemaChangeJob.setStoreRowColumnInfo(hasRowStoreChanged, storeRowColumn, rsColumns);
-        schemaChangeJob.setEnableUniqueKeySkipBitmapInfo(enableUniqueKeySkipBitmap,
-                hasEnableUniqueKeySkipBitmapChanged);
 
         // If StorageFormat is set to TStorageFormat.V2
         // which will create tablet with preferred_rowset_type set to BETA
@@ -2039,22 +2014,6 @@ public int getAsInt() {
                             rowColumn.setUniqueId(maxColUniqueId + 1);
                             indexSchemaMap.get(olapTable.getBaseIndexId()).add(rowColumn);
                         }
-                    } else if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) {
-                        boolean value = Boolean.parseBoolean(properties
-                                .get(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN));
-                        if (!value) {
-                            throw new DdlException("Can not alter enable_unique_key_skip_bitmap_column"
-                                    + " from true to false currently.");
-                        }
-                        if (!olapTable.hasSkipBitmapColumn()) {
-                            Column skipBitmapColumn =
-                                    ColumnDefinition.newSkipBitmapColumnDef(AggregateType.NONE)
-                                            .translateToCatalogStyle();
-                            int maxColUniqueId = olapTable.getIndexMetaByIndexId(olapTable.getBaseIndexId())
-                                    .getMaxColUniqueId();
-                            skipBitmapColumn.setUniqueId(maxColUniqueId + 1);
-                            indexSchemaMap.get(olapTable.getBaseIndexId()).add(skipBitmapColumn);
-                        }
                     }
                 }
 
@@ -3121,17 +3080,9 @@ public void updateBaseIndexSchema(OlapTable olapTable, Map<Long, LinkedList<Colu
         List<Long> indexIds = new ArrayList<Long>();
         indexIds.add(baseIndexId);
         indexIds.addAll(olapTable.getIndexIdListExceptBaseIndex());
-        boolean hasEnableUniqueKeySkipBitmapChanged = false;
         for (int i = 0; i < indexIds.size(); i++) {
             List<Column> indexSchema = indexSchemaMap.get(indexIds.get(i));
             MaterializedIndexMeta currentIndexMeta = olapTable.getIndexMetaByIndexId(indexIds.get(i));
-            if (i == 0 && !olapTable.getEnableUniqueKeySkipBitmap()) {
-                Preconditions.checkState(!currentIndexMeta.getSchema(true).stream()
-                        .anyMatch(col -> col.isSkipBitmapColumn()));
-                if (indexSchema.stream().anyMatch(col -> col.isSkipBitmapColumn())) {
-                    hasEnableUniqueKeySkipBitmapChanged = true;
-                }
-            }
             currentIndexMeta.setSchema(indexSchema);
 
             int currentSchemaVersion = currentIndexMeta.getSchemaVersion();
@@ -3148,9 +3099,6 @@ public void updateBaseIndexSchema(OlapTable olapTable, Map<Long, LinkedList<Colu
             currentIndexMeta.setMaxColUniqueId(maxColUniqueId);
             currentIndexMeta.setIndexes(indexes);
         }
-        if (hasEnableUniqueKeySkipBitmapChanged) {
-            olapTable.setEnableUniqueKeySkipBitmap(true);
-        }
         olapTable.setIndexes(indexes);
         olapTable.rebuildFullSchema();
         olapTable.rebuildDistributionInfo();

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java

@@ -143,11 +143,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
     @SerializedName(value = "hasRowStoreChange")
     protected boolean hasRowStoreChange = false;
 
-    @SerializedName(value = "enableUniqueKeySkipBitmap")
-    private boolean enableUniqueKeySkipBitmap = false;
-    @SerializedName(value = "hasEnableUniqueKeySkipBitmapChanged")
-    private boolean hasEnableUniqueKeySkipBitmapChanged = false;
-
     // save all schema change tasks
     private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask();
 
@@ -201,12 +196,6 @@ public void setStoreRowColumnInfo(boolean hasRowStoreChange,
         this.rowStoreColumns = rowStoreColumns;
     }
 
-    public void setEnableUniqueKeySkipBitmapInfo(boolean enableUniqueKeySkipBitmap,
-            boolean hasEnableUniqueKeySkipBitmapChanged) {
-        this.enableUniqueKeySkipBitmap = enableUniqueKeySkipBitmap;
-        this.hasEnableUniqueKeySkipBitmapChanged = hasEnableUniqueKeySkipBitmapChanged;
-    }
-
     public void setAlterIndexInfo(boolean indexChange, List<Index> indexes) {
         this.indexChange = indexChange;
         this.indexes = indexes;
@@ -770,9 +759,6 @@ private void onFinished(OlapTable tbl) {
             tbl.setStoreRowColumn(storeRowColumn);
             tbl.setRowStoreColumns(rowStoreColumns);
         }
-        if (hasEnableUniqueKeySkipBitmapChanged) {
-            tbl.setEnableUniqueKeySkipBitmap(enableUniqueKeySkipBitmap);
-        }
 
         // set storage format of table, only set if format is v2
         if (storageFormat == TStorageFormat.V2) {

fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java

@@ -158,10 +158,10 @@ public void rewriteAlterClause(OlapTable table) throws UserException {
                         addColumnClause = new AddColumnClause(ColumnDef.newSequenceColumnDef(sequenceColType), null,
                                 null, null);
                     } else if (alterFeature == EnableFeatureClause.Features.UPDATE_FLEXIBLE_COLUMNS) {
-                        ColumnDef skipBItmapCol = ColumnDef.newSkipBitmapColumnDef(AggregateType.NONE);
+                        ColumnDef skipBitmapCol = ColumnDef.newSkipBitmapColumnDef(AggregateType.NONE);
                         List<Column> fullSchema = table.getBaseSchema(true);
                         String lastCol = fullSchema.get(fullSchema.size() - 1).getName();
-                        addColumnClause = new AddColumnClause(skipBItmapCol, new ColumnPosition(lastCol), null, null);
+                        addColumnClause = new AddColumnClause(skipBitmapCol, new ColumnPosition(lastCol), null, null);
                     }
                     addColumnClause.analyze(analyzer);
                     clauses.add(addColumnClause);

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java

@@ -429,7 +429,7 @@ public void analyze(Analyzer analyzer) throws UserException {
             if (properties != null) {
                 if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)
                         && !(keysDesc.getKeysType() == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite)) {
-                    throw new AnalysisException("tablet property enable_unique_key_skip_bitmap_column can"
+                    throw new AnalysisException("table property enable_unique_key_skip_bitmap_column can"
                             + "only be set in merge-on-write unique table.");
                 }
                 // the merge-on-write table must have enable_unique_key_skip_bitmap_column table property

fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java

@@ -375,7 +375,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
             this.needTableStable = false;
             this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
         } else if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) {
-            // do nothing, will be analyzed when creating alter job
+            throw new AnalysisException("You can not modify property 'enable_unique_key_skip_bitmap_column'.");
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
             throw new AnalysisException("You can not modify storage_page_size");
         } else {

fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java

@@ -276,7 +276,6 @@ private static CreateTableStmt buildStatisticsTblStmt(String statsTableName, Lis
             {
                 put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, String.valueOf(
                         Math.max(1, Config.min_replication_num_per_tablet)));
-                put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN, "false");
             }
         };
 

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -2856,15 +2856,8 @@ public void setEnableUniqueKeyMergeOnWrite(boolean speedup) {
         getOrCreatTableProperty().setEnableUniqueKeyMergeOnWrite(speedup);
     }
 
-    public void setEnableUniqueKeySkipBitmap(boolean enable) {
-        getOrCreatTableProperty().setEnableUniqueKeySkipBitmap(enable);
-    }
-
     public boolean getEnableUniqueKeySkipBitmap() {
-        if (tableProperty == null) {
-            return false;
-        }
-        return tableProperty.getEnableUniqueKeySkipBitmap();
+        return hasSkipBitmapColumn();
     }
 
     public boolean getEnableUniqueKeyMergeOnWrite() {

fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java

@@ -631,10 +631,6 @@ public void setEnableUniqueKeyMergeOnWrite(boolean enable) {
         properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, Boolean.toString(enable));
     }
 
-    public void setEnableUniqueKeySkipBitmap(boolean enable) {
-        properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN, Boolean.toString(enable));
-    }
-
     public boolean getEnableUniqueKeySkipBitmap() {
         return Boolean.parseBoolean(properties.getOrDefault(
             PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN, "false"));

fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java

@@ -2701,15 +2701,14 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
         }
         olapTable.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite);
 
-        boolean enableUniqueKeySkipBitmap = false;
         if (keysType == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite) {
             try {
-                enableUniqueKeySkipBitmap = PropertyAnalyzer.analyzeUniqueKeySkipBitmapColumn(properties);
+                // don't store this property, check and remove it from `properties`
+                PropertyAnalyzer.analyzeUniqueKeySkipBitmapColumn(properties);
             } catch (AnalysisException e) {
                 throw new DdlException(e.getMessage());
             }
         }
-        olapTable.setEnableUniqueKeySkipBitmap(enableUniqueKeySkipBitmap);
 
         boolean enableDeleteOnDeletePredicate = false;
         try {

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java

@@ -540,7 +540,7 @@ public void validate(ConnectContext ctx) {
             if (properties != null) {
                 if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)
                         && !(keysType.equals(KeysType.UNIQUE_KEYS) && isEnableMergeOnWrite)) {
-                    throw new AnalysisException("tablet property enable_unique_key_skip_bitmap_column can"
+                    throw new AnalysisException("table property enable_unique_key_skip_bitmap_column can"
                             + "only be set in merge-on-write unique table.");
                 }
                 // the merge-on-write table must have enable_unique_key_skip_bitmap_column table property

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java

@@ -368,7 +368,7 @@ public void validate(ConnectContext ctx) throws UserException {
             this.needTableStable = false;
             this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
         } else if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) {
-            // do nothing, will be analyzed when creating alter job
+            throw new AnalysisException("You can not modify property 'enable_unique_key_skip_bitmap_column'.");
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
             throw new AnalysisException("You can not modify storage_page_size");
         } else {

regression-test/data/query_p0/system/test_table_properties.out

@@ -1,6 +1,6 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !select_check_1 --
-105
+102
 
 -- !select_check_2 --
 internal	test_table_properties_db	duplicate_table	_auto_bucket	false
@@ -16,7 +16,6 @@ internal	test_table_properties_db	duplicate_table	disable_auto_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_mow_light_delete	false
 internal	test_table_properties_db	duplicate_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_unique_key_merge_on_write	false
-internal	test_table_properties_db	duplicate_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	duplicate_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	duplicate_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	duplicate_table	group_commit_interval_ms	10000
@@ -50,7 +49,6 @@ internal	test_table_properties_db	listtable	disable_auto_compaction	false
 internal	test_table_properties_db	listtable	enable_mow_light_delete	false
 internal	test_table_properties_db	listtable	enable_single_replica_compaction	false
 internal	test_table_properties_db	listtable	enable_unique_key_merge_on_write	false
-internal	test_table_properties_db	listtable	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	listtable	file_cache_ttl_seconds	0
 internal	test_table_properties_db	listtable	group_commit_data_bytes	134217728
 internal	test_table_properties_db	listtable	group_commit_interval_ms	10000
@@ -84,7 +82,6 @@ internal	test_table_properties_db	unique_table	disable_auto_compaction	false
 internal	test_table_properties_db	unique_table	enable_mow_light_delete	false
 internal	test_table_properties_db	unique_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	unique_table	enable_unique_key_merge_on_write	true
-internal	test_table_properties_db	unique_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	unique_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	unique_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	unique_table	group_commit_interval_ms	10000
@@ -120,7 +117,6 @@ internal	test_table_properties_db	duplicate_table	disable_auto_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_mow_light_delete	false
 internal	test_table_properties_db	duplicate_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_unique_key_merge_on_write	false
-internal	test_table_properties_db	duplicate_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	duplicate_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	duplicate_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	duplicate_table	group_commit_interval_ms	10000
@@ -154,7 +150,6 @@ internal	test_table_properties_db	unique_table	disable_auto_compaction	false
 internal	test_table_properties_db	unique_table	enable_mow_light_delete	false
 internal	test_table_properties_db	unique_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	unique_table	enable_unique_key_merge_on_write	true
-internal	test_table_properties_db	unique_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	unique_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	unique_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	unique_table	group_commit_interval_ms	10000
@@ -192,7 +187,6 @@ internal	test_table_properties_db	duplicate_table	disable_auto_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_mow_light_delete	false
 internal	test_table_properties_db	duplicate_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_unique_key_merge_on_write	false
-internal	test_table_properties_db	duplicate_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	duplicate_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	duplicate_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	duplicate_table	group_commit_interval_ms	10000

regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_property.groovy

@@ -73,6 +73,11 @@ suite('test_flexible_partial_update_property') {
         }
     }
 
+    test {
+        sql """alter table ${tableName} set ("enable_unique_key_skip_bitmap_column"="true");"""
+        exception "You can not modify property 'enable_unique_key_skip_bitmap_column'."
+    }
+
     doSchemaChange """alter table ${tableName} enable feature "UPDATE_FLEXIBLE_COLUMNS";"""
     show_res = sql "show create table ${tableName}"
     assertTrue(show_res.toString().contains('"enable_unique_key_skip_bitmap_column" = "true"'))