From 6329684f99c3727f3d117e5295879986fc81dbcf Mon Sep 17 00:00:00 2001 From: jacktengg Date: Fri, 15 Nov 2024 16:13:29 +0800 Subject: [PATCH] [wg](chore) rename workload group memory property names --- .../schema_workload_groups_scanner.cpp | 4 +- .../runtime/workload_group/workload_group.cpp | 42 +++++----- .../runtime/workload_group/workload_group.h | 22 +++--- .../org/apache/doris/catalog/SchemaTable.java | 4 +- .../resource/workloadgroup/WorkloadGroup.java | 76 ++++++++++++------- .../workloadgroup/WorkloadGroupMgr.java | 2 +- gensrc/thrift/BackendService.thrift | 4 +- .../workload_manager_p0/test_curd_wlg.groovy | 50 ++++++------ 8 files changed, 112 insertions(+), 92 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp b/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp index 43562a8f52cbf10..481360eee905574 100644 --- a/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp @@ -39,8 +39,8 @@ std::vector SchemaWorkloadGroupsScanner::_s_tbls_colu {"SCAN_THREAD_NUM", TYPE_BIGINT, sizeof(int64_t), true}, {"MAX_REMOTE_SCAN_THREAD_NUM", TYPE_BIGINT, sizeof(int64_t), true}, {"MIN_REMOTE_SCAN_THREAD_NUM", TYPE_BIGINT, sizeof(int64_t), true}, - {"SPILL_THRESHOLD_LOW_WATERMARK", TYPE_VARCHAR, sizeof(StringRef), true}, - {"SPILL_THRESHOLD_HIGH_WATERMARK", TYPE_VARCHAR, sizeof(StringRef), true}, + {"MEMORY_LOW_WATERMARK", TYPE_VARCHAR, sizeof(StringRef), true}, + {"MEMORY_HIGH_WATERMARK", TYPE_VARCHAR, sizeof(StringRef), true}, {"TAG", TYPE_VARCHAR, sizeof(StringRef), true}, {"READ_BYTES_PER_SECOND", TYPE_BIGINT, sizeof(int64_t), true}, {"REMOTE_READ_BYTES_PER_SECOND", TYPE_BIGINT, sizeof(int64_t), true}, diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp index 4a98a4fa8d89922..f9405de12737dc6 100644 --- a/be/src/runtime/workload_group/workload_group.cpp +++ b/be/src/runtime/workload_group/workload_group.cpp @@ -47,8 +47,8 @@ namespace doris { const static std::string MEMORY_LIMIT_DEFAULT_VALUE = "0%"; const static bool ENABLE_MEMORY_OVERCOMMIT_DEFAULT_VALUE = true; const static int CPU_HARD_LIMIT_DEFAULT_VALUE = -1; -const static int SPILL_LOW_WATERMARK_DEFAULT_VALUE = 50; -const static int SPILL_HIGH_WATERMARK_DEFAULT_VALUE = 80; +const static int MEMORY_LOW_WATERMARK_DEFAULT_VALUE = 50; +const static int MEMORY_HIGH_WATERMARK_DEFAULT_VALUE = 80; WorkloadGroup::WorkloadGroup(const WorkloadGroupInfo& wg_info) : WorkloadGroup(wg_info, true) {} @@ -64,8 +64,8 @@ WorkloadGroup::WorkloadGroup(const WorkloadGroupInfo& tg_info, bool need_create_ _scan_thread_num(tg_info.scan_thread_num), _max_remote_scan_thread_num(tg_info.max_remote_scan_thread_num), _min_remote_scan_thread_num(tg_info.min_remote_scan_thread_num), - _spill_low_watermark(tg_info.spill_low_watermark), - _spill_high_watermark(tg_info.spill_high_watermark), + _memory_low_watermark(tg_info.memory_low_watermark), + _memory_high_watermark(tg_info.memory_high_watermark), _scan_bytes_per_second(tg_info.read_bytes_per_second), _remote_scan_bytes_per_second(tg_info.remote_read_bytes_per_second), _need_create_query_thread_pool(need_create_query_thread_pool) { @@ -91,12 +91,12 @@ std::string WorkloadGroup::debug_string() const { "TG[id = {}, name = {}, cpu_share = {}, memory_limit = {}, enable_memory_overcommit = " "{}, version = {}, cpu_hard_limit = {}, scan_thread_num = " "{}, max_remote_scan_thread_num = {}, min_remote_scan_thread_num = {}, " - "spill_low_watermark={}, spill_high_watermark={}, is_shutdown={}, query_num={}, " + "memory_low_watermark={}, memory_high_watermark={}, is_shutdown={}, query_num={}, " "read_bytes_per_second={}, remote_read_bytes_per_second={}]", _id, _name, cpu_share(), PrettyPrinter::print(_memory_limit, TUnit::BYTES), _enable_memory_overcommit ? "true" : "false", _version, cpu_hard_limit(), _scan_thread_num, _max_remote_scan_thread_num, _min_remote_scan_thread_num, - _spill_low_watermark, _spill_high_watermark, _is_shutdown, _query_ctxs.size(), + _memory_low_watermark, _memory_high_watermark, _is_shutdown, _query_ctxs.size(), _scan_bytes_per_second, _remote_scan_bytes_per_second); } @@ -104,14 +104,14 @@ std::string WorkloadGroup::memory_debug_string() const { return fmt::format( "TG[id = {}, name = {}, memory_limit = {}, enable_memory_overcommit = " "{}, weighted_memory_limit = {}, total_mem_used = {}, " - "wg_refresh_interval_memory_growth = {}, spill_low_watermark = {}, " - "spill_high_watermark = {}, version = {}, is_shutdown = {}, query_num = {}]", + "wg_refresh_interval_memory_growth = {}, memory_low_watermark = {}, " + "memory_high_watermark = {}, version = {}, is_shutdown = {}, query_num = {}]", _id, _name, PrettyPrinter::print(_memory_limit, TUnit::BYTES), _enable_memory_overcommit ? "true" : "false", PrettyPrinter::print(_weighted_memory_limit, TUnit::BYTES), PrettyPrinter::print(_total_mem_used, TUnit::BYTES), PrettyPrinter::print(_wg_refresh_interval_memory_growth, TUnit::BYTES), - _spill_low_watermark, _spill_high_watermark, _version, _is_shutdown, + _memory_low_watermark, _memory_high_watermark, _version, _is_shutdown, _query_ctxs.size()); } @@ -137,8 +137,8 @@ void WorkloadGroup::check_and_update(const WorkloadGroupInfo& tg_info) { _scan_thread_num = tg_info.scan_thread_num; _max_remote_scan_thread_num = tg_info.max_remote_scan_thread_num; _min_remote_scan_thread_num = tg_info.min_remote_scan_thread_num; - _spill_low_watermark = tg_info.spill_low_watermark; - _spill_high_watermark = tg_info.spill_high_watermark; + _memory_low_watermark = tg_info.memory_low_watermark; + _memory_high_watermark = tg_info.memory_high_watermark; _scan_bytes_per_second = tg_info.read_bytes_per_second; _remote_scan_bytes_per_second = tg_info.remote_read_bytes_per_second; } else { @@ -396,16 +396,16 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info( min_remote_scan_thread_num = tworkload_group_info.min_remote_scan_thread_num; } - // 12 spill low watermark - int spill_low_watermark = SPILL_LOW_WATERMARK_DEFAULT_VALUE; - if (tworkload_group_info.__isset.spill_threshold_low_watermark) { - spill_low_watermark = tworkload_group_info.spill_threshold_low_watermark; + // 12 memory low watermark + int memory_low_watermark = MEMORY_LOW_WATERMARK_DEFAULT_VALUE; + if (tworkload_group_info.__isset.memory_low_watermark) { + memory_low_watermark = tworkload_group_info.memory_low_watermark; } - // 13 spil high watermark - int spill_high_watermark = SPILL_HIGH_WATERMARK_DEFAULT_VALUE; - if (tworkload_group_info.__isset.spill_threshold_high_watermark) { - spill_high_watermark = tworkload_group_info.spill_threshold_high_watermark; + // 13 memory high watermark + int memory_high_watermark = MEMORY_HIGH_WATERMARK_DEFAULT_VALUE; + if (tworkload_group_info.__isset.memory_high_watermark) { + memory_high_watermark = tworkload_group_info.memory_high_watermark; } // 14 scan io @@ -433,8 +433,8 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info( .scan_thread_num = scan_thread_num, .max_remote_scan_thread_num = max_remote_scan_thread_num, .min_remote_scan_thread_num = min_remote_scan_thread_num, - .spill_low_watermark = spill_low_watermark, - .spill_high_watermark = spill_high_watermark, + .memory_low_watermark = memory_low_watermark, + .memory_high_watermark = memory_high_watermark, .read_bytes_per_second = read_bytes_per_second, .remote_read_bytes_per_second = remote_read_bytes_per_second}; } diff --git a/be/src/runtime/workload_group/workload_group.h b/be/src/runtime/workload_group/workload_group.h index 96b8a36df1cfda1..fb89ed8101ad49e 100644 --- a/be/src/runtime/workload_group/workload_group.h +++ b/be/src/runtime/workload_group/workload_group.h @@ -94,11 +94,11 @@ class WorkloadGroup : public std::enable_shared_from_this { void do_sweep(); - int spill_threshold_low_water_mark() const { - return _spill_low_watermark.load(std::memory_order_relaxed); + int memory_low_watermark() const { + return _memory_low_watermark.load(std::memory_order_relaxed); } - int spill_threashold_high_water_mark() const { - return _spill_high_watermark.load(std::memory_order_relaxed); + int memory_high_watermark() const { + return _memory_high_watermark.load(std::memory_order_relaxed); } void set_weighted_memory_ratio(double ratio); @@ -107,7 +107,7 @@ class WorkloadGroup : public std::enable_shared_from_this { _total_mem_used + _wg_refresh_interval_memory_growth.load() + size; if ((realtime_total_mem_used > ((double)_weighted_memory_limit * - _spill_high_watermark.load(std::memory_order_relaxed) / 100))) { + _memory_high_watermark.load(std::memory_order_relaxed) / 100))) { return false; } else { _wg_refresh_interval_memory_growth.fetch_add(size); @@ -122,10 +122,10 @@ class WorkloadGroup : public std::enable_shared_from_this { auto realtime_total_mem_used = _total_mem_used + _wg_refresh_interval_memory_growth.load(); *is_low_wartermark = (realtime_total_mem_used > ((double)_weighted_memory_limit * - _spill_low_watermark.load(std::memory_order_relaxed) / 100)); + _memory_low_watermark.load(std::memory_order_relaxed) / 100)); *is_high_wartermark = (realtime_total_mem_used > ((double)_weighted_memory_limit * - _spill_high_watermark.load(std::memory_order_relaxed) / 100)); + _memory_high_watermark.load(std::memory_order_relaxed) / 100)); } std::string debug_string() const; @@ -233,8 +233,8 @@ class WorkloadGroup : public std::enable_shared_from_this { std::atomic _scan_thread_num; std::atomic _max_remote_scan_thread_num; std::atomic _min_remote_scan_thread_num; - std::atomic _spill_low_watermark; - std::atomic _spill_high_watermark; + std::atomic _memory_low_watermark; + std::atomic _memory_high_watermark; std::atomic _scan_bytes_per_second {-1}; std::atomic _remote_scan_bytes_per_second {-1}; @@ -282,8 +282,8 @@ struct WorkloadGroupInfo { const int scan_thread_num = 0; const int max_remote_scan_thread_num = 0; const int min_remote_scan_thread_num = 0; - const int spill_low_watermark = 0; - const int spill_high_watermark = 0; + const int memory_low_watermark = 0; + const int memory_high_watermark = 0; const int read_bytes_per_second = -1; const int remote_read_bytes_per_second = -1; // log cgroup cpu info diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index e022cb85296a84d..17942dd04a2102b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -487,8 +487,8 @@ public class SchemaTable extends Table { .column("SCAN_THREAD_NUM", ScalarType.createType(PrimitiveType.BIGINT)) .column("MAX_REMOTE_SCAN_THREAD_NUM", ScalarType.createType(PrimitiveType.BIGINT)) .column("MIN_REMOTE_SCAN_THREAD_NUM", ScalarType.createType(PrimitiveType.BIGINT)) - .column("SPILL_THRESHOLD_LOW_WATERMARK", ScalarType.createVarchar(256)) - .column("SPILL_THRESHOLD_HIGH_WATERMARK", ScalarType.createVarchar(256)) + .column("MEMORY_LOW_WATERMARK", ScalarType.createVarchar(256)) + .column("MEMORY_HIGH_WATERMARK", ScalarType.createVarchar(256)) .column("TAG", ScalarType.createVarchar(256)) .column("READ_BYTES_PER_SECOND", ScalarType.createType(PrimitiveType.BIGINT)) .column("REMOTE_READ_BYTES_PER_SECOND", ScalarType.createType(PrimitiveType.BIGINT)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java index 0d5df7ef1cbba8a..089b464fbe40055 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java @@ -75,9 +75,9 @@ public class WorkloadGroup implements Writable, GsonPostProcessable { public static final String MIN_REMOTE_SCAN_THREAD_NUM = "min_remote_scan_thread_num"; - public static final String SPILL_THRESHOLD_LOW_WATERMARK = "spill_threshold_low_watermark"; + public static final String MEMORY_LOW_WATERMARK = "memory_low_watermark"; - public static final String SPILL_THRESHOLD_HIGH_WATERMARK = "spill_threshold_high_watermark"; + public static final String MEMORY_HIGH_WATERMARK = "memory_high_watermark"; public static final String TAG = "tag"; @@ -90,6 +90,10 @@ public class WorkloadGroup implements Writable, GsonPostProcessable { // later more type and workload may be included in the future. public static final String INTERNAL_TYPE = "internal_type"; + // deprecated + public static final String SPILL_THRESHOLD_LOW_WATERMARK = "spill_threshold_low_watermark"; + public static final String SPILL_THRESHOLD_HIGH_WATERMARK = "spill_threshold_high_watermark"; + // NOTE(wb): all property is not required, some properties default value is set in be // default value is as followed // cpu_share=1024, memory_limit=0%(0 means not limit), enable_memory_overcommit=true @@ -97,14 +101,19 @@ public class WorkloadGroup implements Writable, GsonPostProcessable { .add(CPU_SHARE).add(MEMORY_LIMIT).add(ENABLE_MEMORY_OVERCOMMIT).add(MAX_CONCURRENCY) .add(MAX_QUEUE_SIZE).add(QUEUE_TIMEOUT).add(CPU_HARD_LIMIT).add(SCAN_THREAD_NUM) .add(MAX_REMOTE_SCAN_THREAD_NUM).add(MIN_REMOTE_SCAN_THREAD_NUM) - .add(SPILL_THRESHOLD_LOW_WATERMARK).add(SPILL_THRESHOLD_HIGH_WATERMARK) + .add(MEMORY_LOW_WATERMARK).add(MEMORY_HIGH_WATERMARK) .add(TAG).add(READ_BYTES_PER_SECOND).add(REMOTE_READ_BYTES_PER_SECOND).add(INTERNAL_TYPE).build(); + private static final ImmutableMap DEPRECATED_PROPERTIES_NAME = + new ImmutableMap.Builder() + .put(SPILL_THRESHOLD_LOW_WATERMARK, MEMORY_LOW_WATERMARK) + .put(SPILL_THRESHOLD_HIGH_WATERMARK, MEMORY_HIGH_WATERMARK).build(); + public static final ImmutableMap WORKLOAD_TYPE_MAP = new ImmutableMap.Builder() .put(TWorkloadType.INTERNAL.toString().toLowerCase(), TWorkloadType.INTERNAL.getValue()).build(); - public static final int SPILL_LOW_WATERMARK_DEFAULT_VALUE = 50; - public static final int SPILL_HIGH_WATERMARK_DEFAULT_VALUE = 80; + public static final int MEMORY_LOW_WATERMARK_DEFAULT_VALUE = 50; + public static final int MEMORY_HIGH_WATERMARK_DEFAULT_VALUE = 80; private static final Map ALL_PROPERTIES_DEFAULT_VALUE_MAP = Maps.newHashMap(); @@ -169,19 +178,19 @@ private WorkloadGroup(long id, String name, Map properties, long this.cpuHardLimit = Integer.parseInt(cpuHardLimitStr); this.properties.put(CPU_HARD_LIMIT, cpuHardLimitStr); } - if (properties.containsKey(SPILL_THRESHOLD_LOW_WATERMARK)) { - String lowWatermarkStr = properties.get(SPILL_THRESHOLD_LOW_WATERMARK); + if (properties.containsKey(MEMORY_LOW_WATERMARK)) { + String lowWatermarkStr = properties.get(MEMORY_LOW_WATERMARK); if (lowWatermarkStr.endsWith("%")) { lowWatermarkStr = lowWatermarkStr.substring(0, lowWatermarkStr.length() - 1); } - this.properties.put(SPILL_THRESHOLD_LOW_WATERMARK, lowWatermarkStr); + this.properties.put(MEMORY_LOW_WATERMARK, lowWatermarkStr); } - if (properties.containsKey(SPILL_THRESHOLD_HIGH_WATERMARK)) { - String highWatermarkStr = properties.get(SPILL_THRESHOLD_HIGH_WATERMARK); + if (properties.containsKey(MEMORY_HIGH_WATERMARK)) { + String highWatermarkStr = properties.get(MEMORY_HIGH_WATERMARK); if (highWatermarkStr.endsWith("%")) { highWatermarkStr = highWatermarkStr.substring(0, highWatermarkStr.length() - 1); } - this.properties.put(SPILL_THRESHOLD_HIGH_WATERMARK, highWatermarkStr); + this.properties.put(MEMORY_HIGH_WATERMARK, highWatermarkStr); } if (properties.containsKey(TAG)) { this.properties.put(TAG, properties.get(TAG).toLowerCase()); @@ -402,9 +411,9 @@ private static void checkProperties(Map properties) throws DdlEx } } - int lowWaterMark = SPILL_LOW_WATERMARK_DEFAULT_VALUE; - if (properties.containsKey(SPILL_THRESHOLD_LOW_WATERMARK)) { - String lowVal = properties.get(SPILL_THRESHOLD_LOW_WATERMARK); + int lowWaterMark = MEMORY_LOW_WATERMARK_DEFAULT_VALUE; + if (properties.containsKey(MEMORY_LOW_WATERMARK)) { + String lowVal = properties.get(MEMORY_LOW_WATERMARK); if (lowVal.endsWith("%")) { lowVal = lowVal.substring(0, lowVal.length() - 1); } @@ -416,15 +425,15 @@ private static void checkProperties(Map properties) throws DdlEx lowWaterMark = intValue; } catch (NumberFormatException e) { throw new DdlException( - "The allowed " + SPILL_THRESHOLD_LOW_WATERMARK + "The allowed " + MEMORY_LOW_WATERMARK + " value is an integer value between 1 and 100, but input value is " + lowVal); } } - int highWaterMark = SPILL_HIGH_WATERMARK_DEFAULT_VALUE; - if (properties.containsKey(SPILL_THRESHOLD_HIGH_WATERMARK)) { - String highVal = properties.get(SPILL_THRESHOLD_HIGH_WATERMARK); + int highWaterMark = MEMORY_HIGH_WATERMARK_DEFAULT_VALUE; + if (properties.containsKey(MEMORY_HIGH_WATERMARK)) { + String highVal = properties.get(MEMORY_HIGH_WATERMARK); if (highVal.endsWith("%")) { highVal = highVal.substring(0, highVal.length() - 1); } @@ -436,15 +445,15 @@ private static void checkProperties(Map properties) throws DdlEx highWaterMark = intValue; } catch (NumberFormatException e) { throw new DdlException( - "The allowed " + SPILL_THRESHOLD_HIGH_WATERMARK + "The allowed " + MEMORY_HIGH_WATERMARK + " value is an integer value between 1 and 100, but input value is " + highVal); } } if (lowWaterMark > highWaterMark) { - throw new DdlException(SPILL_THRESHOLD_HIGH_WATERMARK + "(" + highWaterMark + ") should bigger than " - + SPILL_THRESHOLD_LOW_WATERMARK + "(" + lowWaterMark + ")"); + throw new DdlException(MEMORY_HIGH_WATERMARK + "(" + highWaterMark + ") should bigger than " + + MEMORY_LOW_WATERMARK + "(" + lowWaterMark + ")"); } if (properties.containsKey(READ_BYTES_PER_SECOND)) { @@ -652,14 +661,14 @@ public TopicInfo toTopicInfo() { tWorkloadGroupInfo.setMinRemoteScanThreadNum(Integer.parseInt(minRemoteScanThreadNumStr)); } - String spillLowWatermarkStr = properties.get(SPILL_THRESHOLD_LOW_WATERMARK); - if (spillLowWatermarkStr != null) { - tWorkloadGroupInfo.setSpillThresholdLowWatermark(Integer.parseInt(spillLowWatermarkStr)); + String memoryLowWatermarkStr = properties.get(MEMORY_LOW_WATERMARK); + if (memoryLowWatermarkStr != null) { + tWorkloadGroupInfo.setMemoryLowWatermark(Integer.parseInt(memoryLowWatermarkStr)); } - String spillHighWatermarkStr = properties.get(SPILL_THRESHOLD_HIGH_WATERMARK); - if (spillHighWatermarkStr != null) { - tWorkloadGroupInfo.setSpillThresholdHighWatermark(Integer.parseInt(spillHighWatermarkStr)); + String memoryHighWatermarkStr = properties.get(MEMORY_HIGH_WATERMARK); + if (memoryHighWatermarkStr != null) { + tWorkloadGroupInfo.setMemoryHighWatermark(Integer.parseInt(memoryHighWatermarkStr)); } String readBytesPerSecStr = properties.get(READ_BYTES_PER_SECOND); @@ -690,7 +699,18 @@ public void write(DataOutput out) throws IOException { public static WorkloadGroup read(DataInput in) throws IOException { String json = Text.readString(in); - return GsonUtils.GSON.fromJson(json, WorkloadGroup.class); + WorkloadGroup workloadGroup = GsonUtils.GSON.fromJson(json, WorkloadGroup.class); + // "spill_threshold_low_watermark" and "spill_threshold_high_watermark" + // are renamed and deprecated, replace them with "memory_low_watermark" + // and "memory_high_watermark" + for (String key : DEPRECATED_PROPERTIES_NAME.keySet()) { + if (workloadGroup.properties.containsKey(key)) { + String value = workloadGroup.properties.get(key); + workloadGroup.properties.remove(key); + workloadGroup.properties.put(DEPRECATED_PROPERTIES_NAME.get(key), value); + } + } + return workloadGroup; } void setMemLimitPercent(Map props) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java index 31da477912d5e6c..5e61f48bec6f2e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java @@ -85,7 +85,7 @@ public class WorkloadGroupMgr extends MasterDaemon implements Writable, GsonPost .add(WorkloadGroup.QUEUE_TIMEOUT).add(WorkloadGroup.CPU_HARD_LIMIT) .add(WorkloadGroup.SCAN_THREAD_NUM).add(WorkloadGroup.MAX_REMOTE_SCAN_THREAD_NUM) .add(WorkloadGroup.MIN_REMOTE_SCAN_THREAD_NUM) - .add(WorkloadGroup.SPILL_THRESHOLD_LOW_WATERMARK).add(WorkloadGroup.SPILL_THRESHOLD_HIGH_WATERMARK) + .add(WorkloadGroup.MEMORY_LOW_WATERMARK).add(WorkloadGroup.MEMORY_HIGH_WATERMARK) .add(WorkloadGroup.TAG) .add(WorkloadGroup.READ_BYTES_PER_SECOND).add(WorkloadGroup.REMOTE_READ_BYTES_PER_SECOND) .add(QueryQueue.RUNNING_QUERY_NUM).add(QueryQueue.WAITING_QUERY_NUM) diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 533999a853ff382..7f073b2b71095ae 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -265,8 +265,8 @@ struct TWorkloadGroupInfo { 9: optional i32 scan_thread_num 10: optional i32 max_remote_scan_thread_num 11: optional i32 min_remote_scan_thread_num - 12: optional i32 spill_threshold_low_watermark - 13: optional i32 spill_threshold_high_watermark + 12: optional i32 memory_low_watermark + 13: optional i32 memory_high_watermark 14: optional i64 read_bytes_per_second 15: optional i64 remote_read_bytes_per_second 16: optional string tag diff --git a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy index 8eb6b7d92a7eb23..5c3e0aafe768579 100644 --- a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy +++ b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy @@ -441,71 +441,71 @@ suite("test_crud_wlg") { // test workload spill property // 1 create group test { - sql "create workload group if not exists spill_group_test_failed properties ( 'spill_threshold_low_watermark'='90%');" - exception "should bigger than spill_threshold_low_watermark" + sql "create workload group if not exists spill_group_test_failed properties ( 'memory_low_watermark'='90%');" + exception "should bigger than memory_low_watermark" } - sql "create workload group if not exists spill_group_test properties ( 'spill_threshold_low_watermark'='10%','spill_threshold_high_watermark'='10%');" - qt_show_spill_1 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,spill_threshold_low_watermark,spill_threshold_high_watermark from information_schema.workload_groups where name in ('spill_group_test');" + sql "create workload group if not exists spill_group_test properties ( 'memory_low_watermark'='10%','memory_high_watermark'='10%');" + qt_show_spill_1 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,memory_low_watermark,memory_high_watermark from information_schema.workload_groups where name in ('spill_group_test');" test { - sql "create workload group if not exists spill_group_test properties ( 'spill_threshold_low_watermark'='20%','spill_threshold_high_watermark'='10%');" - exception "should bigger than spill_threshold_low_watermark" + sql "create workload group if not exists spill_group_test properties ( 'memory_low_watermark'='20%','memory_high_watermark'='10%');" + exception "should bigger than memory_low_watermark" } // 2 alter low - sql "alter workload group spill_group_test properties ( 'spill_threshold_low_watermark'='5%' );" - qt_show_spill_2 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,spill_threshold_low_watermark,spill_threshold_high_watermark from information_schema.workload_groups where name in ('spill_group_test');" + sql "alter workload group spill_group_test properties ( 'memory_low_watermark'='5%' );" + qt_show_spill_2 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,memory_low_watermark,memory_high_watermark from information_schema.workload_groups where name in ('spill_group_test');" test { - sql "alter workload group spill_group_test properties ( 'spill_threshold_low_watermark'='20%' );" - exception "should bigger than spill_threshold_low_watermark" + sql "alter workload group spill_group_test properties ( 'memory_low_watermark'='20%' );" + exception "should bigger than memory_low_watermark" } test { - sql "alter workload group spill_group_test properties ( 'spill_threshold_low_watermark'='0%' );" + sql "alter workload group spill_group_test properties ( 'memory_low_watermark'='0%' );" exception "value is an integer value between 1 and 100" } test { - sql "alter workload group spill_group_test properties ( 'spill_threshold_low_watermark'='101%' );" + sql "alter workload group spill_group_test properties ( 'memory_low_watermark'='101%' );" exception "value is an integer value between 1 and 100" } test { - sql "create workload group if not exists spill_group_test2 properties ( 'spill_threshold_low_watermark'='0%')" + sql "create workload group if not exists spill_group_test2 properties ( 'memory_low_watermark'='0%')" exception "value is an integer value between 1 and 100" } test { - sql "create workload group if not exists spill_group_test2 properties ( 'spill_threshold_low_watermark'='101%')" + sql "create workload group if not exists spill_group_test2 properties ( 'memory_low_watermark'='101%')" exception "value is an integer value between 1 and 100" } // 3 alter high - sql "alter workload group spill_group_test properties ( 'spill_threshold_high_watermark'='40%' );" - qt_show_spill_3 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,spill_threshold_low_watermark,spill_threshold_high_watermark from information_schema.workload_groups where name in ('spill_group_test');" + sql "alter workload group spill_group_test properties ( 'memory_high_watermark'='40%' );" + qt_show_spill_3 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,memory_low_watermark,memory_high_watermark from information_schema.workload_groups where name in ('spill_group_test');" test { - sql "alter workload group spill_group_test properties ( 'spill_threshold_high_watermark'='1%' );" - exception "should bigger than spill_threshold_low_watermark" + sql "alter workload group spill_group_test properties ( 'memory_high_watermark'='1%' );" + exception "should bigger than memory_low_watermark" } test { - sql "alter workload group spill_group_test properties ( 'spill_threshold_high_watermark'='0%' );" + sql "alter workload group spill_group_test properties ( 'memory_high_watermark'='0%' );" exception "value is an integer value between 1 and 100" } test { - sql "alter workload group spill_group_test properties ( 'spill_threshold_high_watermark'='101%' );" + sql "alter workload group spill_group_test properties ( 'memory_high_watermark'='101%' );" exception "value is an integer value between 1 and 100" } test { - sql "create workload group if not exists spill_group_test2 properties ( 'spill_threshold_high_watermark'='0%')" + sql "create workload group if not exists spill_group_test2 properties ( 'memory_high_watermark'='0%')" exception "value is an integer value between 1 and 100" } test { - sql "create workload group if not exists spill_group_test2 properties ( 'spill_threshold_high_watermark'='101%')" + sql "create workload group if not exists spill_group_test2 properties ( 'memory_high_watermark'='101%')" exception "value is an integer value between 1 and 100" } @@ -744,7 +744,7 @@ suite("test_crud_wlg") { // test default value sql "drop workload group if exists default_val_wg" sql "create workload group default_val_wg properties('enable_memory_overcommit'='true');" - qt_select_default_val_wg_1 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,max_remote_scan_thread_num,min_remote_scan_thread_num,spill_threshold_low_watermark,spill_threshold_high_watermark,tag,read_bytes_per_second,remote_read_bytes_per_second from information_schema.workload_groups where name = 'default_val_wg'" + qt_select_default_val_wg_1 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,max_remote_scan_thread_num,min_remote_scan_thread_num,memory_low_watermark,memory_high_watermark,tag,read_bytes_per_second,remote_read_bytes_per_second from information_schema.workload_groups where name = 'default_val_wg'" sql """ alter workload group default_val_wg properties( @@ -763,7 +763,7 @@ suite("test_crud_wlg") { 'remote_read_bytes_per_second'='10'); """ - qt_select_default_val_wg_2 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,max_remote_scan_thread_num,min_remote_scan_thread_num,spill_threshold_low_watermark,spill_threshold_high_watermark,tag,read_bytes_per_second,remote_read_bytes_per_second from information_schema.workload_groups where name = 'default_val_wg'" + qt_select_default_val_wg_2 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,max_remote_scan_thread_num,min_remote_scan_thread_num,memory_low_watermark,memory_high_watermark,tag,read_bytes_per_second,remote_read_bytes_per_second from information_schema.workload_groups where name = 'default_val_wg'" sql """ alter workload group default_val_wg properties( @@ -783,7 +783,7 @@ suite("test_crud_wlg") { ); """ - qt_select_default_val_wg_3 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,max_remote_scan_thread_num,min_remote_scan_thread_num,spill_threshold_low_watermark,spill_threshold_high_watermark,tag,read_bytes_per_second,remote_read_bytes_per_second from information_schema.workload_groups where name = 'default_val_wg'" + qt_select_default_val_wg_3 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,max_remote_scan_thread_num,min_remote_scan_thread_num,memory_low_watermark,memory_high_watermark,tag,read_bytes_per_second,remote_read_bytes_per_second from information_schema.workload_groups where name = 'default_val_wg'" sql "drop workload group if exists default_val_wg"