From 71a3cc7bb439e24a880eb59f887027c2de6301b5 Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Sun, 5 Jan 2025 14:08:02 +0800 Subject: [PATCH] [chore](conf) Set enable_advance_next_id=true by default (#44790) (#46425) pick #44790 Bind the generation of next_id to physical time to ensure its generation is monotonically increasing, even if we directly overwrite the FE metadata and then restart. This way, it supports a lossy cluster rollback in scenarios where major version upgrades are incompatible. --- .../src/main/java/org/apache/doris/common/Config.java | 2 +- .../doris/analysis/ExportToOutfileLogicalPlanTest.java | 8 ++++++++ .../org/apache/doris/common/proc/IndexesProcNodeTest.java | 2 +- .../org/apache/doris/service/FrontendServiceImplTest.java | 5 ++++- .../java/org/apache/doris/utframe/TestWithFeService.java | 4 ++++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index a9a86601002d5a..4dc2ede72973a6 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2872,7 +2872,7 @@ public class Config extends ConfigBase { "Whether to advance the ID generator after becoming Master to ensure that the id " + "generator will not be rolled back even when metadata is rolled back." }) - public static boolean enable_advance_next_id = false; + public static boolean enable_advance_next_id = true; // The count threshold to do manual GC when doing checkpoint but not enough memory. // Set zero to disable it. diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExportToOutfileLogicalPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExportToOutfileLogicalPlanTest.java index 43256a14a13020..096868276fdd7f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExportToOutfileLogicalPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExportToOutfileLogicalPlanTest.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.load.ExportJob; @@ -47,6 +48,12 @@ public class ExportToOutfileLogicalPlanTest extends TestWithFeService { private String dbName = "testDb"; private String tblName = "table1"; + private final boolean defaultEnableAdvanceNextId = Config.enable_advance_next_id; // backup + + { + enableAdvanceNextId = false; + } + /** * create a database and a table * @@ -65,6 +72,7 @@ protected void runBeforeAll() throws Exception { + "PARTITION p4 VALUES LESS THAN (\"50\")\n" + ")\n" + " distributed by hash(k1) buckets 10\n" + "properties(\"replication_num\" = \"1\");"); + Config.enable_advance_next_id = defaultEnableAdvanceNextId; // restore } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java index 966f6c38b5b783..273915a2d20317 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java @@ -96,7 +96,7 @@ public void testFetchResult() throws AnalysisException { Assert.assertEquals(procResult.getRows().get(3).get(5), "col_4"); Assert.assertEquals(procResult.getRows().get(3).get(11), "NGRAM_BF"); Assert.assertEquals(procResult.getRows().get(3).get(12), "ngram_bf index on col_4"); - Assert.assertEquals(procResult.getRows().get(3).get(13), "(\"gram_size\" = \"3\", \"bf_size\" = \"256\")"); + Assert.assertEquals(procResult.getRows().get(3).get(13), "(\"bf_size\" = \"256\", \"gram_size\" = \"3\")"); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java index 2e5a4e0745631e..36a7447e2e4ea5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java @@ -185,7 +185,10 @@ public void testGetDBNames() throws Exception { TGetDbsResult dbNames = impl.getDbNames(params); Assert.assertEquals(dbNames.getDbs().size(), 2); - Assert.assertEquals(dbNames.getDbs(), Arrays.asList("test", "test_")); + List expected = Arrays.asList("test", "test_"); + dbNames.getDbs().sort(String::compareTo); + expected.sort(String::compareTo); + Assert.assertEquals(dbNames.getDbs(), expected); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index 8e25efdfada439..ea8456179dcbd9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -140,6 +140,8 @@ public abstract class TestWithFeService { protected ConnectContext connectContext; protected boolean needCleanDir = true; protected int lastFeRpcPort = 0; + // make it default to enable_advance_next_id + protected boolean enableAdvanceNextId = Config.enable_advance_next_id; protected static final String DEFAULT_CLUSTER_PREFIX = ""; @@ -152,6 +154,8 @@ public Set getEnableNereidsRules() { @BeforeAll public final void beforeAll() throws Exception { + // this.enableAdvanceNextId may be reset by children classes + Config.enable_advance_next_id = this.enableAdvanceNextId; FeConstants.enableInternalSchemaDb = false; beforeCreatingConnectContext(); connectContext = createDefaultCtx();