From 79fc9bd05c3fa6e5745c1bfd859fc9a15e4df771 Mon Sep 17 00:00:00 2001 From: Philip Vallone Date: Wed, 20 Nov 2024 14:28:49 -0600 Subject: [PATCH 1/2] Add suggested Iceberg engine and workload management config for pilots --- getting-started/templates/pilot-sizing.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/getting-started/templates/pilot-sizing.yaml b/getting-started/templates/pilot-sizing.yaml index 9b341995..61cefe83 100644 --- a/getting-started/templates/pilot-sizing.yaml +++ b/getting-started/templates/pilot-sizing.yaml @@ -8,7 +8,7 @@ # kubernetes cluster nodes. The cluster is intended to be deployed # with the following node topology: # 4 nodes - 8 CPU, 32 GB memory - General node pool -# 2 nodes - 8 CPU, 32 GB memory - Dedicated for Dremio +# 3 nodes - 8 CPU, 32 GB memory - Dedicated for Dremio # # Pilot deployments use managed MongoDB Atlas or MongoDB Enterprise. # As such, the MongoDB resources are not specified in this file. @@ -134,6 +134,13 @@ dataframeservice: maxReplicas: 2 targetCPUUtilizationPercentage: 60 targetMemoryUtilizationPercentage: 70 + + queryEngine: + workloadManagement: + writeQueue: + concurrencyLimit: 5 + optimizeQueue: + concurrencyLimit: 4 sldremio: coordinator: @@ -145,6 +152,16 @@ dataframeservice: cpu: 4 memory: 31000 volumeSize: "50Gi" + engines: + - default + - iceberg + engineOverride: + iceberg: + count: 2 + cpu: 4 + memory: 16384 + heapMemoryOverride: 10000 + directMemoryOverride: 5000 zookeeper: count: 1 # The above is the recommended configuration for pilot deployments. From b2d27ed866764f4ec732f26069c043169f80737e Mon Sep 17 00:00:00 2001 From: Cameron Waterman Date: Mon, 9 Dec 2024 14:50:54 -0600 Subject: [PATCH 2/2] Add placeholder values in systemlink-values.yaml template --- .../templates/systemlink-values.yaml | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/getting-started/templates/systemlink-values.yaml b/getting-started/templates/systemlink-values.yaml index 209e4169..a086439b 100644 --- a/getting-started/templates/systemlink-values.yaml +++ b/getting-started/templates/systemlink-values.yaml @@ -761,6 +761,26 @@ dataframeservice: ## region: "us-east-1" + ## Configure the Dremio query engine + ## + # queryEngine: + ## Configuration for Dremio Workload Management. This is used + ## to isolate operations related to data ingestion from queries. For more + ## information, see https://docs.dremio.com/current/admin/workloads/workload-management/ + # workloadManagement: + ## COPY INTO jobs are targeted to this queue. Increase the concurrency limit + ## to make newly ingested data available for query faster. This may require + ## adding more resources to the engine the query is targeting. + # writeQueue: + # concurrencyLimit: 5 + # engineName: "iceberg" + ## Compaction jobs are targeted to this queue. Increase the concurrency limit + ## to make newly ingested data available for query faster. This may require + ## adding more resources to the engine the query is targeting. + # optimizeQueue: + # concurrencyLimit: 4 + # engineName: "iceberg" + ## Configure Dremio access ## sldremio: @@ -774,12 +794,22 @@ dataframeservice: # coordinator: # cpu: 15 # memory: 122800 - # volumeSize: 128Gi + # volumeSize: 256Gi # executor: - # count: 4 + # count: 3 # cpu: 15 - # memory: 122800 - # volumeSize: 128Gi + # memory: 73728 + # volumeSize: 256Gi + # engines: + # - default + # - iceberg + # engineOverride: + # iceberg: + # count: 4 + # cpu: 9 + # memory: 32768 + # heapMemoryOverride: 20000 + # directMemoryOverride: 10000 ## CPU and memory allocated to each zookeeper pod, expressed in CPU cores and MB respectively. ## Count should correspond with the number of nodes that received the "high.mem" label. # zookeeper: