Merge pull request #170 from toschmidt/main

Update Umbra.

index.html

@@ -986,7 +986,7 @@ <h2>Detailed Comparison</h2>
             const ratio = curr_timing !== null ? (constant_time_add + curr_timing) / (constant_time_add + baseline_timing) : null;
 
             let td = document.createElement('td');
-            td.appendChild(document.createTextNode(curr_timing !== null ? `${curr_timing.toFixed(2)}s (×${ratio.toFixed(2)})` : '☠'));
+            td.appendChild(document.createTextNode(curr_timing !== null ? `${curr_timing.toFixed(3)}s (×${ratio.toFixed(2)})` : '☠'));
 
             colorize(td, ratio);
             tr.appendChild(td);

umbra/benchmark.sh

@@ -1,34 +1,32 @@
 #!/bin/bash
 
+# Ubuntu
 sudo apt-get update
 sudo apt-get install -y postgresql-client gzip
 
+# Amazon Linux
+# yum install nc postgresql15
+
+rm -rf hits.tsv
 wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
 gzip -d hits.tsv.gz
-chmod 777 ~ hits.tsv
+chmod 777 hits.tsv
 
 rm -rf umbra-24-01-03.tar.xz umbra
-wget --no-verbose --continue 'https://db.in.tum.de/~schmidt/umbra-24-01-03.tar.xz'
-tar -xf umbra-24-01-03.tar.xz
+wget --no-verbose --continue 'https://db.in.tum.de/~schmidt/umbra-2024-02-04.tar.xz'
+tar -xf umbra-2024-02-04.tar.xz
 
 rm -rf db
 mkdir db
 
 export USEDIRECTIO=1
 umbra/bin/sql -createdb db/umbra.db <<<"ALTER ROLE postgres WITH LOGIN SUPERUSER PASSWORD 'password';"
 
-umbra/bin/server -createSSLFiles -certFile db/umbra.cert -keyFile db/umbra.pem -address 0.0.0.0 db/umbra.db &> umbra.log &
-sleep 1
-
-psql -h /tmp -U postgres -t < create.sql
-psql -h /tmp -U postgres -t -c '\timing' -c "copy hits from 'hits.tsv' with (format text);"
+time umbra/bin/sql db/umbra.db create.sql
 
 ./run.sh 2>&1 | tee log.txt
 
 du -bcs db/
 
 cat log.txt | grep -oP 'Time: \d+\.\d+ ms' | sed -r -e 's/Time: ([0-9]+\.[0-9]+) ms/\1/' |
     awk '{ if (i % 3 == 0) { printf "[" }; printf $1 / 1000; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }'
-
-killall server
-rm -rf hits.tsv

umbra/create.sql

@@ -107,3 +107,5 @@ create table hits (
     primary key (counterid, eventdate, userid, eventtime, watchid)
 ) with (storage=columnar);
 
+copy hits from 'hits.tsv' with (format text);
+

umbra/results/c6a.4xlarge.json

@@ -1,55 +1,54 @@
 {
     "system": "Umbra",
-    "date": "2024-02-02",
+    "date": "2024-02-05",
     "machine": "c6a.4xlarge, 500gb gp2",
     "cluster_size": 1,
     "tags": ["C++", "column-oriented", "PostgreSQL compatible"],
-    "load_time": 1061.541,
-    "data_size": 97767877183,
-    "comment": "Segmentation fault in the middle of the test",
+    "load_time": 505.913,
+    "data_size": 36477780830,
     "result": [
-[0.209086,0.011738,0.012061],
-[0.286187,0.011852,0.003785],
-[0.295367,0.135114,0.044054],
-[0.895806,0.679204,0.680544],
-[0.702135,0.674911,0.677621],
-[4.0023,0.87053,0.613472],
-[0.11274,0.078905,0.087947],
-[0.012837,0.014259,0.014031],
-[2.729,0.350596,0.341228],
-[0.696983,0.639588,0.669598],
-[0.339289,0.038563,0.043669],
-[0.209976,0.05442,0.054552],
-[0.430673,0.587625,0.768441],
-[0.932235,0.787055,0.780889],
-[1.28999,0.276156,0.297337],
-[0.202715,0.188756,0.192702],
-[0.530036,0.623375,0.608413],
-[0.511773,0.595728,0.563421],
-[1.42062,1.23585,1.192],
-[0.035434,0.013534,0.002076],
-[27.6449,14.838,12.4979],
-[11.6779,6.98523,4.57714],
-[28.1569,20.2952,10.3134],
-[153.316,153.2,154.629],
-[4.08712,1.57023,0.007798],
-[0.010185,0.010279,0.009956],
-[0.038879,0.038326,0.039015],
-[19.3634,7.89121,4.27973],
-[16.7256,10.2004,3.92077],
-[0.303641,0.054993,0.055398],
-[4.58902,0.332033,0.130684],
-[2.79384,1.48084,0.731322],
-[1.38139,1.6159,1.39411],
-[12.9291,9.05383,6.94245],
-[5.3304,5.41123,5.38231],
-[0.57384,0.254843,0.26153],
-[null,null,null],
-[null,null,null],
-[null,null,null],
-[null,null,null],
-[null,null,null],
-[null,null,null],
-[null,null,null]
+        [0.030601,0.010866,0.011101],
+        [0.33904,0.003698,0.003548],
+        [0.645752,0.027352,0.026303],
+        [0.719249,0.026984,0.026267],
+        [0.727638,0.13537,0.133211],
+        [2.0782,0.252201,0.255137],
+        [0.338405,0.027632,0.027376],
+        [0.309864,0.004253,0.004158],
+        [1.61685,0.243389,0.263255],
+        [3.01982,0.51973,0.524606],
+        [1.08709,0.033062,0.032059],
+        [1.46887,0.037179,0.035897],
+        [2.0821,0.216457,0.215829],
+        [3.92123,0.391088,0.395779],
+        [2.48072,0.233635,0.21876],
+        [0.765295,0.18303,0.18472],
+        [3.88406,0.518152,0.541263],
+        [3.8342,0.459446,0.476508],
+        [7.15095,0.868591,0.862418],
+        [0.674124,0.001847,0.001711],
+        [18.1118,0.276739,0.27634],
+        [21.1296,0.085333,0.083386],
+        [35.0997,0.143139,0.141073],
+        [122.717,118.44,116.582],
+        [5.06751,0.009433,0.009186],
+        [2.00484,0.00967,0.009456],
+        [5.07312,0.038766,0.038597],
+        [18.5152,0.269536,0.271205],
+        [13.2374,2.55058,2.54432],
+        [0.465351,0.022607,0.021943],
+        [4.74853,0.13671,0.133112],
+        [7.54723,0.204112,0.20182],
+        [5.16736,1.29808,1.30411],
+        [18.7229,1.4329,1.42222],
+        [18.6891,1.47666,1.50563],
+        [0.706767,0.194858,0.196652],
+        [19.6567,0.01349,0.013724],
+        [12.869,0.008628,0.009058],
+        [1.23511,0.019785,0.019251],
+        [34.5888,0.030742,0.027173],
+        [0.503624,0.015988,0.015614],
+        [0.288441,0.016486,0.016809],
+        [3.59948,0.005299,0.005082]
     ]
 }

umbra/results/c6a.metal.json

@@ -1,56 +1,54 @@
 {
     "system": "Umbra",
-    "date": "2024-02-02",
+    "date": "2024-02-05",
     "machine": "c6a.metal, 500gb gp2",
     "cluster_size": 1,
     "tags": ["C++", "column-oriented", "PostgreSQL compatible"],
-    "load_time": 432.753,
-    "data_size": 104811147008,
-    "comment": "It looks like it caches the data in memory, so the cold run results are incorrect. When I attempt to restart the server, it didn't start due to Segmentation fault.",
-    "hide": true,
+    "load_time": 224.253,
+    "data_size": 37985071698,
     "result": [
-[0.021103,0.014069,0.013841],
-[0.016241,0.015651,0.014398],
-[0.016433,0.019444,0.020535],
-[0.019118,0.022744,0.024424],
-[0.087463,0.116452,0.108668],
-[0.137545,0.143623,0.186976],
-[0.022322,0.018086,0.016414],
-[0.01743,0.01459,0.016214],
-[0.13723,0.145615,0.154707],
-[0.602285,0.614764,0.614396],
-[0.052353,0.055475,0.060419],
-[0.066132,0.068412,0.064872],
-[0.143196,0.178901,0.184571],
-[0.246122,0.285504,0.287426],
-[0.141119,0.172389,0.192775],
-[0.133211,0.124609,0.127582],
-[0.187477,0.219748,0.214415],
-[0.157956,0.187116,0.206256],
-[0.267917,0.337226,0.368166],
-[0.036774,0.019947,0.019828],
-[0.180748,0.088995,0.092018],
-[0.050261,0.056437,0.049835],
-[0.085935,0.08502,0.085339],
-[0.384127,0.283856,0.298186],
-[0.023887,0.025708,0.029501],
-[0.026465,0.022693,0.026492],
-[0.038385,0.043431,0.041599],
-[0.108191,0.095912,0.094505],
-[0.384992,0.369839,0.397459],
-[0.023921,0.019091,0.020356],
-[0.105971,0.103385,0.108233],
-[0.133147,0.127664,0.13925],
-[0.332119,0.366251,0.336184],
-[0.592509,0.652576,0.750742],
-[0.839829,0.668914,0.731482],
-[0.386763,0.131991,0.143009],
-[0.038283,0.033977,0.03078],
-[0.02885,0.021694,0.021639],
-[0.025068,0.018045,0.0184],
-[0.041889,0.041119,0.041826],
-[0.030802,0.021935,0.029303],
-[0.020964,0.022451,0.023481],
-[0.017381,0.020976,0.016867]
+        [0.664962,0.014879,0.013896],
+        [0.86897,0.014264,0.014216],
+        [1.39741,0.019753,0.019888],
+        [1.76497,0.022482,0.022356],
+        [1.80665,0.067752,0.081],
+        [3.30264,0.133111,0.156861],
+        [0.785534,0.0208,0.022284],
+        [0.870046,0.015516,0.016097],
+        [2.66753,0.14137,0.15998],
+        [4.51368,0.689448,0.726916],
+        [2.19478,0.053639,0.059179],
+        [2.57461,0.043306,0.045997],
+        [3.30665,0.164523,0.175388],
+        [4.92596,0.234748,0.273746],
+        [3.70171,0.147932,0.182008],
+        [1.851,0.094632,0.113691],
+        [4.93327,0.243824,0.270263],
+        [4.87596,0.226228,0.210788],
+        [8.02335,0.306983,0.333771],
+        [1.7747,0.017516,0.018097],
+        [19.9175,0.097303,0.081085],
+        [22.3949,0.049008,0.049755],
+        [37.544,0.080033,0.08298],
+        [127.193,0.237656,0.249893],
+        [5.70401,0.02041,0.02158],
+        [2.93367,0.020734,0.021048],
+        [6.12096,0.034546,0.038698],
+        [19.8159,0.110082,0.107088],
+        [14.2241,0.336072,0.36425],
+        [0.663126,0.019808,0.019036],
+        [5.58319,0.082806,0.083288],
+        [8.2146,0.106136,0.093486],
+        [5.46498,0.342316,0.345279],
+        [19.5496,0.631508,0.862308],
+        [19.0042,0.849141,0.840496],
+        [0.303872,0.119839,0.087051],
+        [20.4269,0.029673,0.027629],
+        [14.2258,0.02278,0.019946],
+        [7.9593,0.012017,0.013362],
+        [35.6343,0.039568,0.037401],
+        [2.49159,0.014061,0.013447],
+        [2.00332,0.012786,0.011437],
+        [3.64226,0.016922,0.017801]
     ]
 }

umbra/run.sh

@@ -5,9 +5,24 @@ TRIES=3
 cat queries.sql | while read query; do
     sync
     echo 3 | sudo tee /proc/sys/vm/drop_caches
+    umbra/bin/server -createSSLFiles -certFile db/umbra.cert -keyFile db/umbra.pem -address 0.0.0.0 db/umbra.db &> umbra.log &
+
+    retry_count=0
+    while [ $retry_count -lt 120 ]; do
+        if nc -z localhost 5432; then
+            break
+        fi
+
+        retry_count=$((retry_count+1))
+        sleep 1
+    done
 
     echo "$query";
     for i in $(seq 1 $TRIES); do
         psql -h /tmp -U postgres -t -c '\timing' -c "$query" | grep 'Time'
-    done;
-done;
+    done
+
+
+    killall -9 server
+    sleep 2
+done