Add config to control whether to download in batch

apache · Dec 9, 2024 · c9d5081 · c9d5081
1 parent c4c37cf
commit c9d5081
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 4 deletions.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -250,6 +250,8 @@ DEFINE_mInt32(max_download_speed_kbps, "50000");
 DEFINE_mInt32(download_low_speed_limit_kbps, "50");
 // download low speed time(seconds)
 DEFINE_mInt32(download_low_speed_time, "300");
+// whether to download small files in batch
+DEFINE_mBool(enable_batch_download, "false");
 
 DEFINE_String(sys_log_dir, "");
 DEFINE_String(user_function_dir, "${DORIS_HOME}/lib/udf");

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -298,6 +298,8 @@ DECLARE_mInt32(max_download_speed_kbps);
 DECLARE_mInt32(download_low_speed_limit_kbps);
 // download low speed time(seconds)
 DECLARE_mInt32(download_low_speed_time);
+// whether to download small files in batch.
+DECLARE_mBool(enable_batch_download);
 
 // deprecated, use env var LOG_DIR in be.conf
 DECLARE_String(sys_log_dir);

diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp
@@ -415,7 +415,7 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
         }
 
         std::string address = get_host_port(src.host, src.http_port);
-        if (is_support_batch_download(address).ok()) {
+        if (config::enable_batch_download && is_support_batch_download(address).ok()) {
             // download files via batch api.
             LOG_INFO("remote BE supports batch download, use batch file download")
                     .tag("address", address)
@@ -429,9 +429,15 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
                 continue; // Try another BE
             }
         } else {
-            LOG_INFO("remote BE does not support batch download, use single file download")
-                    .tag("address", address)
-                    .tag("remote_dir", remote_dir);
+            if (config::enable_batch_download) {
+                LOG_INFO("remote BE does not support batch download, use single file download")
+                        .tag("address", address)
+                        .tag("remote_dir", remote_dir);
+            } else {
+                LOG_INFO("batch download is disabled, use single file download")
+                        .tag("address", address)
+                        .tag("remote_dir", remote_dir);
+            }
 
             std::string remote_url_prefix;
             {