Merge pull request #290 from ClickHouse/remove-tier

Remove "tier" in favor of explicitly specifying the number of replicas in the API
ClickHouse · Jan 23, 2025 · 7886a1b · 7886a1b
2 parents cb709a0 + 5a97025
commit 7886a1b
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 39 deletions.
diff --git a/clickhouse-cloud/cloud-api.sh b/clickhouse-cloud/cloud-api.sh
@@ -6,28 +6,28 @@
 
 PROVIDER=${PROVIDER:-aws}
 REGION=${REGION:-eu-central-1}
-TIER=${TIER:-development}
-MEMORY=${MEMORY:-0}
+MEMORY=${MEMORY:-8}
+REPLICAS=${REPLICAS:-2}
 PARALLEL_REPLICA=${PARALLEL_REPLICA:-false}
 
 command -v jq || exit 1
 command -v curl || exit 1
 command -v clickhouse-client || exit 1
 
-echo "Provisioning a service in ${PROVIDER}, region ${REGION}, ${TIER} tier, memory ${MEMORY}, with parallel replicas set to ${PARALLEL_REPLICA}"
+echo "Provisioning a service in ${PROVIDER}, region ${REGION}, memory ${MEMORY}, replicas ${REPLICAS}, with parallel replicas set to ${PARALLEL_REPLICA}"
 
-TMPDIR="${PROVIDER}-${REGION}-${TIER}-${MEMORY}-${PARALLEL_REPLICA}-$$"
+TMPDIR="csp-${PROVIDER}-region-${REGION}-replicas-${REPLICAS}-memory-${MEMORY}-parallel-${PARALLEL_REPLICA}-pid-$$"
 mkdir -p "${TMPDIR}"
 
 echo $TMPDIR
 
 curl -X POST -H 'Content-Type: application/json' -d '
 {
-    "name": "ClickBench-'${PROVIDER}'-'${REGION}'-'${TIER}'-'${MEMORY}'-'$$'",
-    "tier": "'$TIER'",
+    "name": "ClickBench-'${PROVIDER}'-'${REGION}'-'${REPLICAS}'-'${MEMORY}'-'$$'",
     "provider": "'$PROVIDER'",
     "region": "'$REGION'",
-    '$([ $TIER == production ] && echo -n "\"minTotalMemoryGb\":${MEMORY},\"maxTotalMemoryGb\":${MEMORY},")'
+    "numReplicas": '$REPLICAS',
+    "minReplicaMemoryGb":'$MEMORY',"maxReplicaMemoryGb":'$MEMORY',
     "ipAccessList": [{"source": "0.0.0.0/0", "description": "anywhere"}]
 }
 ' --silent --show-error --user "${KEY_ID}:${KEY_SECRET}" "https://api.clickhouse.cloud/v1/organizations/${ORGANIZATION}/services" | tee "${TMPDIR}"/service.json | jq | grep -v password
@@ -69,8 +69,8 @@ do
 done
 
 if [ ${PARALLEL_REPLICA} = true ]; then
-   echo "Enabling parallel replica to the default user"
-   clickhouse-client --host "$FQDN" --password "$PASSWORD" --secure --query "ALTER USER default SETTINGS allow_experimental_parallel_reading_from_replicas=2;"
+    echo "Enabling parallel replica to the default user"
+    clickhouse-client --host "$FQDN" --password "$PASSWORD" --secure --query "ALTER USER default SETTINGS enable_parallel_replicas = 1"
 fi
 
 echo "Running the benchmark"

diff --git a/clickhouse-cloud/collect-results.sh b/clickhouse-cloud/collect-results.sh
@@ -6,15 +6,17 @@
 for f in */result
 do
     echo $f
-    PROVIDER=$(echo "$f" | grep -o -E '^[a-z]+')
-    MACHINE=$(echo "$f" | sed -r -e 's~^[a-z0-9-]+-([0-9]+)-[a-z]+-[0-9]+/.*$~\1~; s/^0/dev/; s/([0-9]+)/\1GB/')
+    REGEXP='^csp-([a-z0-9-]+)-region-([a-z0-9-]+)-replicas-([a-z0-9-]+)-memory-([a-z0-9-]+)-parallel-([a-z0-9-]+)-pid-([a-z0-9-]+)/result$'
+    PROVIDER=$(echo "$f" | sed -r -e 's!'$REGEXP'!\1!')
+    REPLICAS=$(echo "$f" | sed -r -e 's!'$REGEXP'!\3!')
+    MEMORY=$(echo "$f" | sed -r -e 's!'$REGEXP'!\4!')
 
     echo '
 {
     "system": "ClickHouse Cloud ('$PROVIDER')",
     "date": "'$(date +%F)'",
-    "machine": "'$MACHINE'",
-    "cluster_size": "serverless",
+    "machine": "'$MEMORY'GiB",
+    "cluster_size": "'$REPLICAS'",
     "comment": "",
 
     "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "'$PROVIDER'"],
@@ -26,5 +28,5 @@ do
 '$(grep -F "[" "$f" | head -c-2)'
 ]
 }
-' > "results/${PROVIDER}.${MACHINE}.json"
+' > "results/${PROVIDER}.${REPLICAS}.${MEMORY}.json"
 done
diff --git a/clickhouse-cloud/combinations.sh b/clickhouse-cloud/combinations.sh
@@ -8,43 +8,70 @@ PROVIDER=aws
 REGION='us-east-1'
 PARALLEL_REPLICA=false
 
-TIER=development
-MEMORY=0
-
-export PROVIDER TIER REGION MEMORY PARALLEL_REPLICA
-./cloud-api.sh &
+for REPLICAS in 1
+do
+    for MEMORY in 8 12
+    do
+        export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
+        ./cloud-api.sh &
+        sleep 1 # Prevent "Too many requests" to the API
+    done
+done
 
-TIER=production
-for MEMORY in 24 48 96 192 360 720
+for REPLICAS in 2 3
 do
-    export PROVIDER TIER REGION MEMORY PARALLEL_REPLICA
-    ./cloud-api.sh &
+    for MEMORY in 8 12 16 32 64 120 236
+    do
+        export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
+        ./cloud-api.sh &
+        sleep 1
+    done
 done
 
 PROVIDER=gcp
 REGION='us-east1'
 
-TIER=development
-MEMORY=0
-
-export PROVIDER TIER REGION MEMORY PARALLEL_REPLICA
-./cloud-api.sh &
+for REPLICAS in 1
+do
+    for MEMORY in 8 12
+    do
+        export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
+        ./cloud-api.sh &
+        sleep 1
+    done
+done
 
-TIER=production
-for MEMORY in 24 48 96 192 360 708
+for REPLICAS in 2 3
 do
-    export PROVIDER TIER REGION MEMORY PARALLEL_REPLICA
-    ./cloud-api.sh &
+    for MEMORY in 8 12 16 32 64 120 236
+    do
+        export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
+        ./cloud-api.sh &
+        sleep 1
+    done
 done
 
 PROVIDER=azure
 REGION='eastus2'
 
-TIER=production
-for MEMORY in 24 48 96 192 360
+for REPLICAS in 1
+do
+    for MEMORY in 8 12
+    do
+        export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
+        ./cloud-api.sh &
+        sleep 1
+    done
+done
+
+for REPLICAS in 2 3
 do
-    export PROVIDER TIER REGION MEMORY PARALLEL_REPLICA
-    ./cloud-api.sh &
+    for MEMORY in 8 12 16 32 64 120
+    do
+        export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
+        ./cloud-api.sh &
+        sleep 1
+    done
 done
 
 wait
diff --git a/clickhouse-cloud/run.sh b/clickhouse-cloud/run.sh
@@ -5,8 +5,8 @@ QUERY_NUM=1
 cat queries.sql | while read -r query; do
     echo -n "["
     for i in $(seq 1 $TRIES); do
-        clickhouse-client --host "${FQDN:=localhost}" --password "${PASSWORD:=}" ${PASSWORD:+--secure} --time --format=Null --query="$query" --progress 0 2>&1 |
-            grep -o -P '^\d+\.\d+$' | tr -d '\n' || echo -n "null"
+        (clickhouse-client --host "${FQDN:=localhost}" --password "${PASSWORD:=}" ${PASSWORD:+--secure} --time --format=Null --query="$query" --progress 0 2>&1 |
+            grep -o -P '^\d+\.\d+$' || echo -n "null") | tr -d '\n'
         [[ "$i" != $TRIES ]] && echo -n ", "
     done
     echo "],"

diff --git a/generate-results.sh b/generate-results.sh
@@ -19,7 +19,7 @@ fi
         [[ $file =~ ^(hardware|versions|gravitons)/ ]] && continue;
 
         [ "${FIRST}" = "0" ] && echo -n ','
-        jq --compact-output ". += {\"source\": \"${file}\"}" "${file}"
+        jq --compact-output ". += {\"source\": \"${file}\"}" "${file}" || echo "Error in $file" >&2
         FIRST=0
     done