Add results

clickhouse-cloud/cloud-api.sh

@@ -15,30 +15,33 @@ command -v clickhouse-client || exit 1
 
 echo "Provisioning a service in ${PROVIDER}, region ${REGION}, ${TIER} tier, memory ${MEMORY}"
 
+TMPDIR="${PROVIDER}-${REGION}-${TIER}-${MEMORY}-$$"
+mkdir -p "${TMPDIR}"
+
 curl -X POST -H 'Content-Type: application/json' -d '
 {
-    "name": "ClickBench-'${PROVIDER}'-'${REGION}'-'${TIER}'-'${MEMORY}'",
+    "name": "ClickBench-'${PROVIDER}'-'${REGION}'-'${TIER}'-'${MEMORY}'-'$$'",
     "tier": "'$TIER'",
     "provider": "'$PROVIDER'",
     "region": "'$REGION'",
     '$([ $TIER == production ] && echo -n "\"minTotalMemoryGb\":${MEMORY},\"maxTotalMemoryGb\":${MEMORY},")'
     "ipAccessList": [{"source": "0.0.0.0/0", "description": "anywhere"}]
 }
-' --silent --show-error --user "${KEY_ID}:${KEY_SECRET}" "https://api.clickhouse.cloud/v1/organizations/${ORGANIZATION}/services" | tee service.json | jq
+' --silent --show-error --user "${KEY_ID}:${KEY_SECRET}" "https://api.clickhouse.cloud/v1/organizations/${ORGANIZATION}/services" | tee "${TMPDIR}"/service.json | jq
 
-[ $(jq .status service.json) != 200 ] && exit 1
+[ $(jq .status "${TMPDIR}"/service.json) != 200 ] && exit 1
 
-export SERVICE_ID=$(jq --raw-output .result.service.id service.json)
-export FQDN=$(jq --raw-output .result.service.endpoints[0].host service.json)
-export PASSWORD=$(jq --raw-output .result.password service.json)
+export SERVICE_ID=$(jq --raw-output .result.service.id "${TMPDIR}"/service.json)
+export FQDN=$(jq --raw-output .result.service.endpoints[0].host "${TMPDIR}"/service.json)
+export PASSWORD=$(jq --raw-output .result.password "${TMPDIR}"/service.json)
 
 echo "Waiting for it to start"
 
 for i in {0..1000}
 do
     echo -n "$i seconds... "
-    curl --silent --show-error --user $KEY_ID:$KEY_SECRET "https://api.clickhouse.cloud/v1/organizations/${ORGANIZATION}/services/${SERVICE_ID}" | jq --raw-output .result.state | tee state
-    grep 'running' state && break
+    curl --silent --show-error --user $KEY_ID:$KEY_SECRET "https://api.clickhouse.cloud/v1/organizations/${ORGANIZATION}/services/${SERVICE_ID}" | jq --raw-output .result.state | tee "${TMPDIR}"/state
+    grep 'running' "${TMPDIR}"/state && break
     sleep 1
 done
 
@@ -52,7 +55,7 @@ done
 
 echo "Running the benchmark"
 
-./benchmark.sh
+./benchmark.sh 2>&1 | tee "${TMPDIR}"/result
 
 echo "Stopping the service"
 
@@ -63,8 +66,8 @@ echo "Waiting for the service to stop"
 for i in {0..1000}
 do
     echo -n "$i seconds... "
-    curl --silent --show-error --user $KEY_ID:$KEY_SECRET "https://api.clickhouse.cloud/v1/organizations/${ORGANIZATION}/services/${SERVICE_ID}" | jq --raw-output .result.state | tee state
-    grep 'stopped' state && break
+    curl --silent --show-error --user $KEY_ID:$KEY_SECRET "https://api.clickhouse.cloud/v1/organizations/${ORGANIZATION}/services/${SERVICE_ID}" | jq --raw-output .result.state | tee "${TMPDIR}"/state
+    grep 'stopped' "${TMPDIR}"/state && break
     sleep 1
 done
 

clickhouse-cloud/collect-results.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+for f in */result
+do
+    PROVIDER=$(echo "$f" | grep -oP '^\w+')
+    MACHINE=$(echo "$f" | sed -r -e 's!^[a-z0-9-]+-([0-9]+)-[0-9]+/.+$!\1!; s/^0/dev/; s/([0-9]+)/\1GB/')
+
+    echo '
+{
+    "system": "ClickHouse Cloud ('$PROVIDER')",
+    "date": "'$(date +%F)'",
+    "machine": "'$MACHINE'",
+    "cluster_size": "serverless",
+    "comment": "",
+
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "'$PROVIDER'"],
+
+    "load_time": '$(head -n1 "$f" | tr -d "\n")',
+    "data_size": '$(tail -n1 "$f" | tr -d "\n")',
+
+    "result": [
+'$(grep -F "[" "$f" | head -c-2)'
+]
+}
+' > "results/${PROVIDER}.${MACHINE}.json"
+done

clickhouse-cloud/combinations.sh

@@ -4,24 +4,32 @@
 # export KEY_ID=...
 # export KEY_SECRET=...
 
-for PROVIDER in aws gcp
-do
-    if [ "$PROVIDER" == 'aws' ]
-    then
-        REGION='eu-central-1'
-    else
-        REGION='europe-west-4'
-    fi
+PROVIDER=aws
+REGION='eu-central-1'
+
+TIER=development
+MEMORY=0
+export PROVIDER TIER REGION MEMORY
+./cloud-api.sh &
 
-    TIER=development
-    MEMORY=0
+TIER=production
+for MEMORY in 24 48 96 192 360 720
+do
     export PROVIDER TIER REGION MEMORY
-    ./cloud-api.sh
+    ./cloud-api.sh &
+done
 
-    TIER=production
-    for MEMORY in 24 48 96 192 360 720
-    do
-        export PROVIDER TIER REGION MEMORY
-        ./cloud-api.sh
-    done
+PROVIDER=gcp
+REGION='europe-west4'
+
+TIER=development
+MEMORY=0
+export PROVIDER TIER REGION MEMORY
+./cloud-api.sh &
+
+TIER=production
+for MEMORY in 24 48 96 192 360 708
+do
+    export PROVIDER TIER REGION MEMORY
+    ./cloud-api.sh &
 done

clickhouse-cloud/results/aws.192GB.json

@@ -0,0 +1,18 @@
+
+{
+    "system": "ClickHouse Cloud (aws)",
+    "date": "2023-09-03",
+    "machine": "192GB",
+    "cluster_size": "serverless",
+    "comment": "",
+
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "aws"],
+
+    "load_time": 224.417,
+    "data_size": 9940946872,
+
+    "result": [
+[0.003, 0.004, 0.003], [0.770, 0.067, 0.022], [0.351, 0.690, 0.045], [0.762, 0.072, 0.586], [0.324, 0.432, 0.361], [0.861, 0.697, 0.786], [0.509, 0.054, 0.031], [0.035, 0.542, 0.394], [0.986, 0.624, 0.640], [0.705, 0.658, 0.621], [0.295, 0.245, 0.191], [0.346, 0.335, 0.295], [0.706, 0.808, 0.809], [0.856, 0.857, 0.968], [0.836, 0.861, 0.983], [0.431, 0.434, 0.387], [1.550, 1.564, 1.692], [1.379, 1.084, 1.405], [3.511, 3.560, 3.617], [0.072, 0.134, 0.035], [1.324, 2.146, 1.026], [1.294, 1.591, 1.237], [2.004, 1.217, 2.248], [4.849, 6.715, 3.885], [0.432, 0.354, 0.354], [0.365, 0.367, 0.309], [0.351, 0.440, 0.394], [1.106, 1.224, 0.971], [8.116, 9.850, 9.576], [1.224, 1.211, 0.820], [0.690, 0.646, 0.699], [0.865, 0.847, 0.746], [3.477, 2.954, 2.843], [2.562, 2.985, 2.519], [2.451, 2.462, 2.528], [0.967, 0.918, 0.883], [0.095, 0.089, 0.091], [0.049, 0.057, 0.148], [0.138, 0.072, 0.070], [0.299, 0.195, 0.218], [0.058, 0.329, 0.047], [0.021, 0.045, 0.027], [0.038, 0.029, 0.033]
+]
+}
+

clickhouse-cloud/results/aws.24GB.json

@@ -0,0 +1,18 @@
+
+{
+    "system": "ClickHouse Cloud (aws)",
+    "date": "2023-09-03",
+    "machine": "24GB",
+    "cluster_size": "serverless",
+    "comment": "",
+
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "aws"],
+
+    "load_time": 898.272,
+    "data_size": 9949087959,
+
+    "result": [
+[0.002, 0.002, 0.002], [0.134, 0.151, 1.040], [0.481, 0.547, 0.228], [0.952, 0.427, 0.560], [2.505, 2.465, 2.421], [3.252, 3.143, 3.498], [0.281, 0.121, 0.139], [0.048, 0.042, 0.043], [4.113, 2.525, 2.291], [2.742, 2.965, 2.652], [1.161, 1.069, 1.193], [1.555, 1.333, 1.357], [3.277, 3.667, 3.194], [4.850, 5.542, 5.120], [4.442, 4.498, 4.674], [2.645, 2.745, 2.731], [11.061, 10.301, 10.394], [8.293, 7.391, 7.995], [32.636, 30.847, 30.519], [0.099, 0.133, 0.129], [7.746, 9.111, 5.625], [13.131, 6.666, 6.920], [24.838, 12.782, 8.131], [33.065, 23.814, 46.946], [1.903, 1.852, 2.352], [1.172, 1.294, 1.241], [1.913, 1.883, 1.851], [6.512, 5.868, 6.225], [58.901, 55.853, 55.978], [3.891, 5.452, 4.000], [2.982, 3.194, 3.433], [5.779, 4.125, 4.880], [32.403, 35.532, 34.985], [27.241, 27.673, 26.393], [26.661, 25.919, 28.050], [4.740, 4.937, 4.659], [0.361, 0.175, 0.168], [0.079, 0.101, 0.070], [0.177, 0.088, 0.072], [0.336, 0.341, 0.327], [0.231, 0.033, 0.039], [0.056, 0.192, 0.027], [0.023, 0.021, 0.026]
+]
+}
+

clickhouse-cloud/results/aws.360GB.json

@@ -0,0 +1,18 @@
+
+{
+    "system": "ClickHouse Cloud (aws)",
+    "date": "2023-09-03",
+    "machine": "360GB",
+    "cluster_size": "serverless",
+    "comment": "",
+
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "aws"],
+
+    "load_time": 216.929,
+    "data_size": 9942386190,
+
+    "result": [
+[0.003, 0.005, 0.003], [0.651, 0.021, 0.647], [0.307, 0.039, 0.293], [0.638, 0.628, 0.140], [0.273, 0.303, 0.259], [0.585, 0.521, 0.431], [0.117, 0.025, 0.066], [0.059, 0.032, 0.042], [0.610, 0.540, 1.024], [1.158, 1.010, 0.534], [0.236, 0.234, 0.223], [0.289, 0.210, 0.249], [0.444, 0.455, 0.454], [0.582, 0.592, 0.644], [0.739, 0.586, 0.513], [0.313, 0.320, 0.369], [1.126, 1.175, 1.242], [0.838, 0.828, 0.792], [1.880, 2.022, 2.056], [0.036, 0.058, 0.043], [0.950, 0.921, 1.028], [0.842, 0.832, 0.827], [0.931, 1.435, 0.904], [2.660, 2.337, 5.094], [0.258, 0.270, 0.235], [0.305, 0.214, 0.231], [0.257, 0.261, 0.238], [0.770, 1.022, 0.812], [5.886, 4.937, 5.757], [0.655, 0.627, 0.638], [0.427, 0.675, 0.476], [0.541, 0.595, 0.696], [2.303, 2.386, 1.514], [1.604, 1.685, 1.776], [1.590, 1.616, 1.653], [0.621, 0.722, 0.490], [0.174, 0.156, 0.093], [0.084, 0.046, 0.059], [0.061, 0.132, 0.083], [0.283, 0.315, 0.207], [0.062, 0.316, 0.037], [0.210, 0.037, 0.032], [0.029, 0.038, 0.052]
+]
+}
+

clickhouse-cloud/results/aws.48GB.json

@@ -0,0 +1,18 @@
+
+{
+    "system": "ClickHouse Cloud (aws)",
+    "date": "2023-09-03",
+    "machine": "48GB",
+    "cluster_size": "serverless",
+    "comment": "",
+
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "aws"],
+
+    "load_time": 313.530,
+    "data_size": 9941379875,
+
+    "result": [
+[0.002, 0.002, 0.002], [0.575, 0.147, 0.015], [0.248, 0.775, 0.090], [0.789, 0.388, 0.179], [1.162, 1.033, 1.021], [1.732, 1.498, 2.214], [0.128, 0.052, 0.079], [0.058, 0.030, 0.047], [1.560, 1.619, 1.152], [1.958, 1.490, 1.820], [0.498, 0.453, 0.450], [0.643, 0.797, 0.615], [1.818, 1.617, 2.116], [2.826, 2.798, 2.674], [2.364, 2.222, 2.019], [1.275, 1.221, 1.322], [5.230, 4.694, 5.178], [3.506, 3.187, 2.962], [9.025, 11.452, 10.064], [0.098, 0.058, 0.090], [4.267, 3.073, 3.884], [2.856, 3.725, 2.821], [6.760, 3.852, 3.267], [15.132, 10.307, 11.082], [0.956, 0.905, 0.933], [0.607, 0.724, 0.662], [0.915, 0.917, 0.888], [2.453, 2.441, 2.577], [22.966, 24.132, 22.600], [1.990, 1.856, 2.246], [1.629, 1.615, 1.535], [2.691, 2.884, 1.926], [14.293, 15.228, 13.532], [6.949, 7.111, 9.260], [7.081, 8.422, 8.627], [3.143, 2.628, 2.513], [0.193, 0.129, 0.155], [0.048, 0.051, 0.086], [0.121, 0.070, 0.076], [0.370, 0.210, 0.179], [0.044, 0.242, 0.040], [0.042, 0.031, 0.042], [0.151, 0.031, 0.024]
+]
+}
+

clickhouse-cloud/results/aws.720GB.json

@@ -0,0 +1,18 @@
+
+{
+    "system": "ClickHouse Cloud (aws)",
+    "date": "2023-09-03",
+    "machine": "720GB",
+    "cluster_size": "serverless",
+    "comment": "",
+
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "aws"],
+
+    "load_time": 220.239,
+    "data_size": 9941859054,
+
+    "result": [
+[0.004, 0.004, 0.005], [0.118, 0.059, 0.637], [0.248, 0.166, 0.036], [0.175, 0.516, 0.501], [0.222, 0.192, 0.195], [0.405, 0.262, 0.264], [0.164, 0.054, 0.026], [0.058, 0.421, 0.064], [0.522, 0.411, 0.549], [0.552, 0.451, 0.457], [0.245, 0.180, 0.210], [0.196, 0.211, 0.178], [0.514, 0.266, 0.571], [0.662, 0.635, 0.593], [0.398, 0.400, 0.365], [0.236, 0.223, 0.343], [0.665, 0.631, 0.652], [0.474, 0.526, 0.469], [1.462, 1.219, 1.468], [0.032, 0.027, 0.055], [0.762, 1.088, 0.477], [0.553, 0.502, 0.507], [0.928, 0.618, 0.706], [4.070, 3.194, 1.992], [0.206, 0.163, 0.147], [0.128, 0.177, 0.114], [0.177, 0.171, 0.172], [0.528, 0.647, 0.600], [4.037, 4.033, 4.040], [0.663, 0.629, 0.650], [0.286, 0.302, 0.299], [0.395, 0.366, 0.335], [1.704, 1.802, 1.760], [1.307, 1.265, 1.361], [1.276, 1.281, 1.380], [0.351, 0.317, 0.319], [0.134, 0.120, 0.122], [0.070, 0.053, 0.064], [0.086, 0.082, 0.087], [0.171, 0.180, 0.204], [0.063, 0.034, 0.046], [0.059, 0.050, 0.044], [0.061, 0.049, 0.039]
+]
+}
+

clickhouse-cloud/results/aws.96GB.json

@@ -0,0 +1,18 @@
+
+{
+    "system": "ClickHouse Cloud (aws)",
+    "date": "2023-09-03",
+    "machine": "96GB",
+    "cluster_size": "serverless",
+    "comment": "",
+
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "aws"],
+
+    "load_time": 227.755,
+    "data_size": 9942057937,
+
+    "result": [
+[0.002, 0.002, 0.002], [0.709, 0.039, 0.317], [0.163, 0.295, 0.754], [0.330, 0.144, 0.813], [0.661, 0.869, 0.654], [1.315, 1.402, 1.182], [0.177, 0.042, 0.070], [0.586, 0.052, 0.042], [1.178, 1.141, 1.472], [1.062, 1.060, 0.875], [0.605, 0.374, 0.403], [0.492, 0.443, 0.409], [1.228, 1.300, 1.254], [2.161, 1.633, 1.644], [1.493, 1.734, 1.602], [0.986, 0.939, 0.823], [3.121, 3.165, 2.713], [1.867, 1.716, 3.116], [5.606, 7.876, 5.516], [0.168, 0.262, 0.084], [2.063, 1.632, 1.636], [2.995, 3.168, 1.861], [2.431, 4.313, 2.309], [12.271, 6.228, 7.296], [0.751, 0.699, 0.659], [0.489, 0.475, 0.563], [0.692, 0.592, 0.554], [1.750, 1.626, 1.768], [14.788, 15.976, 15.280], [1.456, 1.202, 1.483], [0.985, 1.104, 1.059], [2.162, 1.552, 1.099], [4.401, 6.886, 6.793], [4.445, 4.203, 4.891], [4.964, 4.319, 4.943], [1.400, 1.475, 1.427], [0.101, 0.120, 0.119], [0.080, 0.088, 0.048], [0.122, 0.067, 0.047], [0.214, 0.186, 0.263], [0.137, 0.025, 0.026], [0.157, 0.057, 0.042], [0.023, 0.058, 0.037]
+]
+}
+

clickhouse-cloud/results/aws.dev.json

@@ -0,0 +1,18 @@
+
+{
+    "system": "ClickHouse Cloud (aws)",
+    "date": "2023-09-03",
+    "machine": "dev",
+    "cluster_size": "serverless",
+    "comment": "",
+
+    "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "aws"],
+
+    "load_time": 700.114,
+    "data_size": 9944443104,
+
+    "result": [
+[0.002, 0.002, 0.002], [0.114, 0.112, 0.044], [0.530, 0.223, 0.231], [0.658, 0.321, 0.578], [2.042, 1.725, 1.454], [2.508, 2.023, 2.725], [0.091, 0.091, 0.084], [0.033, 0.081, 0.035], [1.891, 2.362, 2.009], [2.071, 1.951, 2.251], [1.077, 0.666, 0.809], [1.082, 0.890, 0.977], [2.813, 3.022, 2.817], [3.666, 3.734, 3.593], [3.647, 3.371, 2.700], [2.513, 1.743, 1.641], [6.708, 9.375, 8.844], [4.948, 6.079, 5.460], [26.159, 20.860, 17.466], [0.395, 0.080, 0.107], [3.658, 3.448, 6.485], [6.897, 6.948, 4.022], [8.765, 7.751, 4.769], [13.487, 27.999, 20.242], [1.132, 1.961, 1.199], [0.799, 0.705, 0.838], [2.009, 1.063, 1.744], [3.712, 5.554, 3.618], [32.808, 60.019, 32.276], [3.389, 2.354, 3.351], [1.779, 2.202, 2.205], [3.687, 3.694, 2.223], [23.809, 16.334, 16.007], [19.995, 19.951, 17.978], [16.470, 19.988, 19.945], [3.228, 2.638, 3.916], [0.303, 0.103, 0.105], [0.257, 0.093, 0.064], [0.080, 0.136, 0.066], [0.270, 0.275, 0.347], [0.055, 0.205, 0.056], [0.182, 0.059, 0.033], [0.082, 0.176, 0.030]
+]
+}
+