diff --git a/be/src/clucene b/be/src/clucene index 5a458e6112b7e50..7cf6cf410d41d95 160000 --- a/be/src/clucene +++ b/be/src/clucene @@ -1 +1 @@ -Subproject commit 5a458e6112b7e5010262594147adf22830b096e6 +Subproject commit 7cf6cf410d41d95456edba263cc55b7b6f5ab027 diff --git a/be/src/olap/partial_update_info.cpp b/be/src/olap/partial_update_info.cpp index 33b520d9a496c25..b138677e49cae8e 100644 --- a/be/src/olap/partial_update_info.cpp +++ b/be/src/olap/partial_update_info.cpp @@ -564,6 +564,12 @@ Status FlexibleReadPlan::fill_non_primary_key_columns_for_column_store( assert_cast( new_col.get()) ->insert_null_elements(1); + } else if (tablet_column.is_auto_increment()) { + // In flexible partial update, the skip bitmap indicates whether a cell + // is specified in the original load, so the generated auto-increment value is filled + // in current block in place if needed rather than using a seperate column to + // store the generated auto-increment value in fixed partial update + new_col->insert_from(cur_col, block_pos); } else { new_col->insert_default(); } @@ -640,6 +646,12 @@ Status FlexibleReadPlan::fill_non_primary_key_columns_for_row_store( assert_cast( new_col.get()) ->insert_null_elements(1); + } else if (tablet_column.is_auto_increment()) { + // In flexible partial update, the skip bitmap indicates whether a cell + // is specified in the original load, so the generated auto-increment value is filled + // in current block in place if needed rather than using a seperate column to + // store the generated auto-increment value in fixed partial update + new_col->insert_from(cur_col, block_pos); } else { new_col->insert_default(); } diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index 4db2d62b9949e77..cd3163cd1a9c536 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -1855,12 +1855,7 @@ void NewJsonReader::_process_skip_bitmap_mark(SlotDescriptor* slot_desc, IColumn assert_cast(skip_bitmap_nullable_col_ptr->get_nested_column_ptr().get()); DCHECK(skip_bitmap_col_ptr->size() == cur_row_count + 1); auto& skip_bitmap = skip_bitmap_col_ptr->get_data().back(); - if (!slot_desc->is_auto_increment()) { - // For auto-increment column, it will always have a valid value when in SegmentWriter. - // Either the row specifies it, or its value is filled with generated value. So never mark the - // auto-increment column in skip bitmap - skip_bitmap.add(slot_desc->col_unique_id()); - } + skip_bitmap.add(slot_desc->col_unique_id()); } void NewJsonReader::_collect_profile_before_close() { diff --git a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.out b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.out index ccdf59b7f7b1d82..b5e2c1d647a10e7 100644 --- a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.out +++ b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.out @@ -11,11 +11,11 @@ 11 -- !autoinc_key_3 -- -\N 111 111 1234 \N 1,4,5,6 -\N 9876 666 1234 \N 1,2,4,6 -\N 9876 888 222 \N 1,2,5,6 -43 9876 99 20 \N 2,5,6 -999 9876 777 1234 \N 2,4,5,6 +\N 111 111 1234 \N 0,1,4,5,6 +\N 9876 666 1234 \N 0,1,2,4,6 +\N 9876 888 222 \N 0,1,2,5,6 +43 9876 99 20 \N 0,2,5,6 +999 9876 777 1234 \N 0,2,4,5,6 -- !autoinc_key_4 -- 1000 10 0 0 0 0 2,3,4,5,6 @@ -35,7 +35,7 @@ -- !autoinc_val_2 -- 0 0 0 0 0 0 -1 10 1 1 1 1 2,4,5,6 +1 10 1 1 1 1 2,3,4,5,6 2 2 20 20 2 25 1,4,6 3 3 3 30 3 3 1,2,4,5,6 4 4 4 4 4 4 @@ -43,8 +43,8 @@ 9 \N 99 77 1234 \N 1,4,5,6 -- !autoinc_val_3 -- -8 \N 88 1234 \N 1,4,5,6 -10 \N 9876 666 555 1,2,6 +8 \N 88 1234 \N 1,3,4,5,6 +10 \N 9876 666 555 1,2,3,6 -- !autoinc_val_4 -- 2 @@ -61,11 +61,11 @@ 11 -- !autoinc_key_3 -- -\N 111 111 1234 \N 1,4,5,6 -\N 9876 666 1234 \N 1,2,4,6 -\N 9876 888 222 \N 1,2,5,6 -43 9876 99 20 \N 2,5,6 -999 9876 777 1234 \N 2,4,5,6 +\N 111 111 1234 \N 0,1,4,5,6 +\N 9876 666 1234 \N 0,1,2,4,6 +\N 9876 888 222 \N 0,1,2,5,6 +43 9876 99 20 \N 0,2,5,6 +999 9876 777 1234 \N 0,2,4,5,6 -- !autoinc_key_4 -- 1000 10 0 0 0 0 2,3,4,5,6 @@ -85,7 +85,7 @@ -- !autoinc_val_2 -- 0 0 0 0 0 0 -1 10 1 1 1 1 2,4,5,6 +1 10 1 1 1 1 2,3,4,5,6 2 2 20 20 2 25 1,4,6 3 3 3 30 3 3 1,2,4,5,6 4 4 4 4 4 4 @@ -93,8 +93,8 @@ 9 \N 99 77 1234 \N 1,4,5,6 -- !autoinc_val_3 -- -8 \N 88 1234 \N 1,4,5,6 -10 \N 9876 666 555 1,2,6 +8 \N 88 1234 \N 1,3,4,5,6 +10 \N 9876 666 555 1,2,3,6 -- !autoinc_val_4 -- 2 diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.groovy index 93e56b8a85b00e4..bd9f69fc3ddc908 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.groovy @@ -92,5 +92,30 @@ suite('test_flexible_partial_update_auto_inc') { qt_autoinc_val_2 "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} where k not in (8,10) order by k;" qt_autoinc_val_3 "select k,v1,v2,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} where k in (8,10) order by k;" qt_autoinc_val_4 "select count(distinct v3) from ${tableName} where k in (8,10);" + + def originAutoIncVals = [] + def res1 = sql_return_maparray "select k,v1,v2,v3,v4,v5 from ${tableName} where k in (8,10)"; + for (def row : res1) { + originAutoIncVals << row.v3 + } + Collections.sort(originAutoIncVals) + logger.info("originAutoIncVals; ${originAutoIncVals}") + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "autoinc2.json" + time 20000 + } + def autoIncVals = [] + def res2 = sql_return_maparray "select k,v1,v2,v3,v4,v5 from ${tableName} where k in (8,10)"; + for (def row : res1) { + autoIncVals << row.v3 + } + Collections.sort(autoIncVals) + logger.info("autoIncVals: ${autoIncVals}") + assertEquals(originAutoIncVals, autoIncVals) } } \ No newline at end of file