Merge pull request #123 from adam8157/tune_greenplum

Tune the Greenplum

greenplum/benchmark.sh

@@ -12,8 +12,9 @@ sudo apt update
 sudo apt install greenplum-db-6
 sudo rm -rf /gpmaster /gpdata*
 ssh-keygen -t rsa -b 4096
+touch /home/gpadmin/.ssh/authorized_keys
+chmod 600 ~/.ssh/authorized_keys
 cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys
-mod 600 ~/.ssh/authorized_keys
 sudo echo "# kernel.shmall = _PHYS_PAGES / 2 # See Shared Memory Pages
 kernel.shmall = 197951838
 # kernel.shmmax = kernel.shmall * PAGE_SIZE 
@@ -53,7 +54,7 @@ echo "* soft nofile 524288
 echo "RemoveIPC=no" |sudo tee -a  /etc/systemd/logind.conf
 echo "Now you need to reboot the machine. Press Enter if you already rebooted, or reboot now and run the script once again"
 read
-source /opt/greenplum-db-*.0/greenplum_path.sh
+source /opt/greenplum-db-*/greenplum_path.sh
 cp $GPHOME/docs/cli_help/gpconfigs/gpinitsystem_singlenode .
 echo localhost > ./hostlist_singlenode
 sed -i "s/MASTER_HOSTNAME=[a-z_]*/MASTER_HOSTNAME=$(hostname)/" gpinitsystem_singlenode
@@ -62,12 +63,13 @@ sudo mkdir /gpmaster /gpdata1 /gpdata2 /gpdata3 /gpdata4 /gpdata5 /gpdata6 /gpda
 sudo chmod 777 /gpmaster /gpdata1 /gpdata2 /gpdata3 /gpdata4 /gpdata5 /gpdata6 /gpdata7 /gpdata8 /gpdata9 /gpdata10 /gpdata11 /gpdata12 /gpdata13 /gpdata14
 gpinitsystem -ac gpinitsystem_singlenode
 export MASTER_DATA_DIRECTORY=/gpmaster/gpsne-1/
-#wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-#gzip -d hits.tsv.gz
+wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+gzip -d hits.tsv.gz
 chmod 777 ~ hits.tsv
 psql -d postgres -f create.sql
 nohup gpfdist &
 time psql -d postgres -t -c '\timing' -c "insert into hits select * from hits_ext;"
+psql -d postgres -t -c "ANALYZE hits;"
 du -sh /gpdata*
 ./run.sh 2>&1 | tee log.txt
 cat log.txt | grep -oP 'Time: \d+\.\d+ ms' | sed -r -e 's/Time: ([0-9]+\.[0-9]+) ms/\1/' |awk '{ if (i % 3 == 0) { printf "[" }; printf $1 / 1000; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }'

greenplum/create.sql

@@ -108,7 +108,7 @@ CREATE TABLE hits
     CLID INTEGER NOT NULL
 )
 with (appendoptimized=true,orientation=column,compresstype=zstd)
-DISTRIBUTED RANDOMLY;
+DISTRIBUTED BY (UserID);
 CREATE INDEX hits_idx on hits using btree (CounterID, EventDate, UserID, EventTime, WatchID); 
 drop external table if exists hits_ext;
 CREATE EXTERNAL TABLE hits_ext (like hits)

greenplum/results/c6a.4xlarge.json

@@ -1,6 +1,6 @@
 {
     "system": "Greenplum",
-    "date": "2022-07-01",
+    "date": "2022-08-18",
     "machine": "c6a.4xlarge, 500gb gp2",
     "cluster_size": 1,
     "comment": "",
@@ -11,48 +11,48 @@
     "data_size": 42000000000,
 
     "result": [
-[2.20009,1.45883,1.13573],
-[1.28661,1.07897,1.01856],
-[2.2582,2.42193,2.59693],
-[1.89371,1.9246,1.73866],
-[9.54471,9.71956,10.0357],
-[4.55957,4.10112,4.70458],
-[1.65167,1.89767,1.97051],
-[1.02537,0.999323,1.00524],
-[7.89792,7.75718,8.01219],
-[11.3779,11.6003,11.2552],
-[2.11955,2.05003,1.9553],
-[2.01997,1.97932,1.98284],
-[2.88666,2.92828,2.96822],
-[4.06272,4.01771,4.03011],
-[3.46303,3.24779,3.06475],
-[8.43717,8.18641,7.99202],
-[9.02176,8.91572,8.92748],
-[8.66982,8.39332,8.62497],
-[15.1321,14.9502,15.2344],
-[1.00877,0.934589,0.975342],
-[11.0582,3.25807,3.17687],
-[12.2634,3.5188,3.93705],
-[24.1522,4.86328,4.88764],
-[67.1517,20.4,20.261],
-[2.5799,1.66114,1.90432],
-[1.72831,1.89536,1.2515],
-[2.44154,1.46595,1.88238],
-[11.1175,6.09551,6.02292],
-[83.041,82.9784,82.8678],
-[82.5441,82.5427,83.9836],
-[3.0962,3.22225,3.19431],
-[5.77576,4.05973,4.00366],
-[43.6826,40.4375,40.1076],
-[18.0978,13.2741,12.8894],
-[17.3187,13.5411,13.5927],
-[9.72405,9.90029,10.0177],
-[0.294344,0.169606,0.173804],
-[0.162524,0.117489,0.115532],
-[0.145205,0.098342,0.097275],
-[0.334809,0.275365,0.265053],
-[0.154522,0.107654,0.10529],
-[0.158957,0.117284,0.119068],
-[0.193756,0.144787,0.145485]
-]
+	[2.19644,1.27581,1.07334],
+	[1.36821,1.09289,1.00081],
+	[1.84455,1.93433,1.9734],
+	[1.38533,1.07306,0.997511],
+	[10.4301,10.1377,10.0064],
+	[3.24183,3.67479,3.23526],
+	[1.81177,1.22447,1.72696],
+	[1.28568,1.12341,1.02112],
+	[3.5159,3.7946,3.51106],
+	[5.78795,5.90487,5.95635],
+	[1.50773,1.80496,1.26395],
+	[1.68682,1.93425,1.54081],
+	[2.29378,2.75715,2.22458],
+	[3.41812,3.09848,3.0303],
+	[2.96859,2.96836,2.9949],
+	[3.04409,3.00935,2.98848],
+	[4.10668,3.98626,4.00533],
+	[3.51045,3.14617,3.02558],
+	[9.31981,9.03239,9.00459],
+	[0.563991,0.853628,0.971569],
+	[8.40649,3.29997,3.82986],
+	[9.52108,3.52917,3.86565],
+	[18.4842,4.56174,4.67819],
+	[50.449,16.4463,16.4923],
+	[1.96056,1.59658,1.85278],
+	[1.76567,1.51262,1.52955],
+	[2.11338,1.81927,1.92422],
+	[8.6392,5.18686,5.77986],
+	[51.8749,52.0756,52.2289],
+	[63.6736,63.868,63.3526],
+	[2.92731,2.88967,2.96908],
+	[4.83141,3.88826,3.41858],
+	[42.2016,40.1165,39.4692],
+	[14.5369,10.3602,10.4522],
+	[14.7224,10.829,10.8651],
+	[5.32702,5.79151,5.20852],
+	[0.339799,0.158694,0.160555],
+	[0.173487,0.112253,0.112419],
+	[0.158136,0.102158,0.101214],
+	[0.341324,0.263589,0.24586],
+	[0.171119,0.109181,0.108795],
+	[0.178617,0.117175,0.117738],
+	[0.205935,0.137013,0.137252]
+    ]
 }