From 30ef984afa6479f9a3a5bff01325dfeeeff0a3d8 Mon Sep 17 00:00:00 2001 From: w41ter Date: Tue, 5 Nov 2024 07:04:00 +0000 Subject: [PATCH] [improve](report) split agent batch tasks automaticlly to avoid the binary message size exceeds the thrift max message size limitation --- .../src/main/java/org/apache/doris/common/Config.java | 8 ++++++++ .../main/java/org/apache/doris/master/ReportHandler.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index dd0aca5923e74a4..81b7f6929151c75 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1208,6 +1208,14 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int report_queue_size = 100; + // if the number of report task in FE exceed max_report_task_num_per_rpc, then split it to multiple rpc + @ConfField(mutable = true, masterOnly = true, description = { + "重新发送 agent task 时,单次 RPC 分配给每个be的任务最大个数,默认值为10000个。", + "The max number of batched tasks per RPC assigned to each be when resend agent tasks, " + + "the default value is 10000." + }) + public static int report_resend_batch_task_num_per_rpc = 10000; + /** * If set to true, metric collector will be run as a daemon timer to collect metrics at fix interval */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 1c8f51bd4ebf244..bc51590ccb5a786 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -619,7 +619,7 @@ private static void taskReport(long backendId, Map> running List diffTasks = AgentTaskQueue.getDiffTasks(backendId, runningTasks); - AgentBatchTask batchTask = new AgentBatchTask(); + AgentBatchTask batchTask = new AgentBatchTask(Config.report_resend_batch_task_num_per_rpc); long taskReportTime = System.currentTimeMillis(); for (AgentTask task : diffTasks) { // these tasks no need to do diff