[chore](backup) Remove delete_if_exists properties for creating repos…

…itory delete_if_exists is a temporary solution introduced in apache#25847, to avoid concurrent testing conflicts. Cherry-pick apache#38190
w41ter · Jul 22, 2024 · a07d5e4 · a07d5e4
1 parent ce71a13
commit a07d5e4
Show file tree

Hide file tree

Showing 16 changed files with 20 additions and 53 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
@@ -28,8 +28,6 @@
 import java.util.Map;
 
 public class CreateRepositoryStmt extends DdlStmt {
-    public static String PROP_DELETE_IF_EXISTS = "delete_if_exists";
-
     private boolean isReadOnly;
     private String name;
     private StorageBackend storage;
@@ -73,16 +71,6 @@ public void analyze(Analyzer analyzer) throws UserException {
             ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
         }
         FeNameFormat.checkCommonName("repository", name);
-
-        // check delete_if_exists, this property will be used by Repository.initRepository.
-        Map<String, String> properties = getProperties();
-        String deleteIfExistsStr = properties.get(PROP_DELETE_IF_EXISTS);
-        if (deleteIfExistsStr != null) {
-            if (!deleteIfExistsStr.equalsIgnoreCase("true") && !deleteIfExistsStr.equalsIgnoreCase("false")) {
-                ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
-                        "'" + PROP_DELETE_IF_EXISTS + "' in properties, you should set it false or true");
-            }
-        }
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
@@ -17,7 +17,6 @@
 
 package org.apache.doris.backup;
 
-import org.apache.doris.analysis.CreateRepositoryStmt;
 import org.apache.doris.analysis.StorageBackend;
 import org.apache.doris.backup.Status.ErrCode;
 import org.apache.doris.catalog.Env;
@@ -221,26 +220,6 @@ public Status initRepository() {
             return Status.OK;
         }
 
-        // A temporary solution is to delete all stale snapshots before creating an S3 repository
-        // so that we can add regression tests about backup/restore.
-        //
-        // TODO: support hdfs/brokers
-        if (fileSystem instanceof S3FileSystem) {
-            String deleteStaledSnapshots = fileSystem.getProperties()
-                    .getOrDefault(CreateRepositoryStmt.PROP_DELETE_IF_EXISTS, "false");
-            if (deleteStaledSnapshots.equalsIgnoreCase("true")) {
-                // delete with prefix:
-                // eg. __palo_repository_repo_name/
-                String snapshotPrefix = Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name));
-                LOG.info("property {} is set, delete snapshots with prefix: {}",
-                        CreateRepositoryStmt.PROP_DELETE_IF_EXISTS, snapshotPrefix);
-                Status st = ((S3FileSystem) fileSystem).deleteDirectory(snapshotPrefix);
-                if (!st.ok()) {
-                    return st;
-                }
-            }
-        }
-
         String repoInfoFilePath = assembleRepoInfoFilePath();
         // check if the repo is already exist in remote
         List<RemoteFile> remoteFiles = Lists.newArrayList();

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
@@ -818,8 +818,7 @@ class Syncer {
             "s3.endpoint" = "http://${endpoint}",
             "s3.region" = "${region}",
             "s3.access_key" = "${ak}",
-            "s3.secret_key" = "${sk}",
-            "delete_if_exists" = "true"
+            "s3.secret_key" = "${sk}"
         )
             """
     }

diff --git a/regression-test/suites/backup_restore/test_backup_restore.groovy b/regression-test/suites/backup_restore/test_backup_restore.groovy
@@ -16,9 +16,11 @@
 // under the License.
 
 suite("test_backup_restore", "backup_restore") {
-    String repoName = "test_backup_restore_repo"
-    String dbName = "backup_restore_db"
-    String tableName = "test_backup_restore_table"
+    String suiteName = "test_backup_restore"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
+    String dbName = "${suiteName}_db"
+    String tableName = "${suiteName}_table"
+    String snapshotName = "${suiteName}_snapshot"
 
     def syncer = getSyncer()
     syncer.createS3Repository(repoName)
@@ -45,7 +47,6 @@ suite("test_backup_restore", "backup_restore") {
     def result = sql "SELECT * FROM ${dbName}.${tableName}"
     assertEquals(result.size(), values.size());
 
-    String snapshotName = "test_backup_restore_snapshot"
     sql """
         BACKUP SNAPSHOT ${dbName}.${snapshotName}
         TO `${repoName}`

diff --git a/regression-test/suites/backup_restore/test_backup_restore_alias.groovy b/regression-test/suites/backup_restore/test_backup_restore_alias.groovy
@@ -16,7 +16,7 @@
 // under the License.
 
 suite("test_backup_restore_alias", "backup_restore") {
-    String repoName = "test_backup_restore_alias_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String dbName = "backup_restore_alias_db"
     String tableName = "test_backup_restore_alias_table"
     String aliasName = "test_backup_restore_alias_table_alias"

diff --git a/regression-test/suites/backup_restore/test_backup_restore_db.groovy b/regression-test/suites/backup_restore/test_backup_restore_db.groovy
@@ -18,7 +18,7 @@
 suite("test_backup_restore_db", "backup_restore") {
     String dbName = "backup_restore_db_1"
     String suiteName = "test_backup_restore_db"
-    String repoName = "${suiteName}_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String snapshotName = "${suiteName}_snapshot"
     String tableNamePrefix = "${suiteName}_tables"
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy b/regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy
@@ -17,7 +17,7 @@
 
 suite("test_backup_restore_diff_repo_same_snapshot", "backup_restore") {
     String suiteName = "test_backup_restore_diff_repo_same_snapshot"
-    String repoName = "${suiteName}_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String dbName = "${suiteName}_db"
     String tableName = "${suiteName}_table"
     String snapshotName = "${suiteName}_snapshot"

diff --git a/regression-test/suites/backup_restore/test_backup_restore_dup_without_default_keys.groovy b/regression-test/suites/backup_restore/test_backup_restore_dup_without_default_keys.groovy
@@ -16,7 +16,7 @@
 // under the License.
 
 suite("test_backup_restore_dup_without_default_keys", "backup_restore") {
-    String repoName = "test_backup_restore_dup_without_default_keys_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String dbName = "backup_restore_dup_without_default_keys_db"
     String tableName = "dup_without_keys_table"
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy b/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
@@ -18,7 +18,7 @@
 suite("test_backup_restore_exclude", "backup_restore") {
     String dbName = "backup_restore_exclude"
     String suiteName = "test_backup_restore_exclude"
-    String repoName = "${suiteName}_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String snapshotName = "${suiteName}_snapshot"
     String tableNamePrefix = "${suiteName}_tables"
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy b/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy
@@ -18,7 +18,7 @@
 suite("test_backup_restore_multi_tables", "backup_restore") {
     String dbName = "backup_restore_multi_tables_db"
     String suiteName = "test_backup_restore_multi_tables"
-    String repoName = "${suiteName}_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String snapshotName = "${suiteName}_snapshot"
     String tableNamePrefix = "${suiteName}_tables"
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy b/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
@@ -18,7 +18,7 @@
 suite("test_backup_restore_multi_tables_overwrite", "backup_restore") {
     String dbName = "backup_restore_multi_tables_overwrite_db"
     String suiteName = "test_backup_restore_multi_tables_overwrite"
-    String repoName = "${suiteName}_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String snapshotName = "${suiteName}_snapshot"
     String tableNamePrefix = "${suiteName}_tables"
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_with_two_phase_fetch_opt.groovy b/regression-test/suites/backup_restore/test_backup_restore_with_two_phase_fetch_opt.groovy
@@ -17,7 +17,7 @@
 
 suite("test_backup_restore_with_two_phase_fetch_opt", "backup_restore") {
     String suiteName = "test_backup_restore_with_two_phase_fetch_opt"
-    String repoName = "${suiteName}_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String dbName = "${suiteName}_db"
     String tableName = "${suiteName}_table"
     String snapshotName = "${suiteName}_snapshot"

diff --git a/regression-test/suites/backup_restore/test_duplicate_backup_restore.groovy b/regression-test/suites/backup_restore/test_duplicate_backup_restore.groovy
@@ -16,7 +16,7 @@
 // under the License.
 
 suite("test_duplicate_backup_restore", "backup_restore") {
-    String repoName = "test_duplicate_backup_restore_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String dbName = "duplicate_backup_restore_db"
     String tableName = "test_duplicate_backup_restore_table"
 
@@ -85,7 +85,7 @@ suite("test_duplicate_backup_restore", "backup_restore") {
     sql "DELETE FROM ${dbName}.${tableName} WHERE id = 1"
     result = sql "SELECT * FROM ${dbName}.${tableName}"
     assertEquals(result.size(), values.size() + 5 - 3)
-    repoName = "test_duplicate_delete_backup_restore_repo"
+    repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     syncer.createS3Repository(repoName)
     snapshotName = "test_duplicate_delete_backup_restore_snapshot"
     sql """

diff --git a/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy b/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy
@@ -18,7 +18,7 @@
 suite("test_restore_mix_exists_and_new_table", "backup_restore") {
     String dbName = "restore_mix_exists_and_new_table"
     String suiteName = "test_restore_mix_exists_and_new_table"
-    String repoName = "${suiteName}_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String snapshotName = "${suiteName}_snapshot"
     String tableNamePrefix = "${suiteName}_tables"
 

diff --git a/regression-test/suites/backup_restore/test_restore_to_new_table.groovy b/regression-test/suites/backup_restore/test_restore_to_new_table.groovy
@@ -16,7 +16,7 @@
 // under the License.
 
 suite("test_restore_to_new_table", "backup_restore") {
-    String repoName = "test_restore_to_new_table_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String dbName = "restore_to_new_table_db"
     String tableName = "test_restore_to_new_table_table"
     String snapshotName = "test_backup_restore_snapshot"

diff --git a/regression-test/suites/backup_restore/test_unique_backup_restore.groovy b/regression-test/suites/backup_restore/test_unique_backup_restore.groovy
@@ -16,7 +16,7 @@
 // under the License.
 
 suite("test_unique_backup_restore", "backup_restore") {
-    String repoName = "test_unique_backup_restore_repo"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     String dbName = "unique_backup_restore_db"
     String tableName = "test_unique_backup_restore_table"
 
@@ -85,7 +85,7 @@ suite("test_unique_backup_restore", "backup_restore") {
     sql "DELETE FROM ${dbName}.${tableName} WHERE id = 1"
     result = sql "SELECT * FROM ${dbName}.${tableName}"
     assertEquals(result.size(), 1)
-    repoName = "test_unique_delete_backup_restore_repo"
+    repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
     syncer.createS3Repository(repoName)
     snapshotName = "test_unique_delete_backup_restore_snapshot"
     sql """