Test for checking results aws

Added query generator which generates equivalent queries for aws and ceph. After runing these queries, their results are matched. Signed-off-by: Girjesh Rajoria <[email protected]>
ceph · Mar 26, 2021 · f3f022a · f3f022a
1 parent b95da97
commit f3f022a
Show file tree

Hide file tree

Showing 18 changed files with 535 additions and 0 deletions.
diff --git a/test/queries_generator/aws_cmds.sh b/test/queries_generator/aws_cmds.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+set -x
+set -e
+
+mkdir -p aws_results
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select lower(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from s3object;" "aws_results/output1.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select to_timestamp('1999-11-04T20:27:03.479340Z') from s3object;" "aws_results/output2.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select date_add(day, 8, to_timestamp('1982-02-03T00:20:12.42831Z')) from s3object;" "aws_results/output3.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select substring(upper('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  '), cast((avg(cast(_3 as int)-cast(_2 as int)) - 3) as int), min(cast(_1 as int)-cast(_1 as int)) + 7) from s3object;" "aws_results/output4.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select 10-cast(_2 as int) from s3object;" "aws_results/output5.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select character_length(to_string(to_timestamp('1963-06-22T05:35:39.954350Z'), 'SSSSS HH ')) from s3object;" "aws_results/output6.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select cast(_2 as int)-2 from s3object;" "aws_results/output7.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select char_length(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from s3object;" "aws_results/output8.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select to_string(date_add(second, 5, to_timestamp('1938-09-24T10:57:42.142042Z')), 'hh ') from s3object;" "aws_results/output9.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select to_timestamp('1931-12-23T02:54:25.451925Z') from s3object;" "aws_results/output10.csv"
diff --git a/test/queries_generator/aws_queries.txt b/test/queries_generator/aws_queries.txt
@@ -0,0 +1,10 @@
+select lower(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from s3object;
+select to_timestamp('1999-11-04T20:27:03.479340Z') from s3object;
+select date_add(day, 8, to_timestamp('1982-02-03T00:20:12.42831Z')) from s3object;
+select substring(upper('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  '), cast((avg(cast(_3 as int)-cast(_2 as int)) - 3) as int), min(cast(_1 as int)-cast(_1 as int)) + 7) from s3object;
+select 10-cast(_2 as int) from s3object;
+select character_length(to_string(to_timestamp('1963-06-22T05:35:39.954350Z'), 'SSSSS HH ')) from s3object;
+select cast(_2 as int)-2 from s3object;
+select char_length(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from s3object;
+select to_string(date_add(second, 5, to_timestamp('1938-09-24T10:57:42.142042Z')), 'hh ') from s3object;
+select to_timestamp('1931-12-23T02:54:25.451925Z') from s3object;
diff --git a/test/queries_generator/aws_results/output1.csv b/test/queries_generator/aws_results/output1.csv
@@ -0,0 +1,10 @@
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
diff --git a/test/queries_generator/aws_results/output10.csv b/test/queries_generator/aws_results/output10.csv
@@ -0,0 +1,10 @@
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
diff --git a/test/queries_generator/aws_results/output2.csv b/test/queries_generator/aws_results/output2.csv
@@ -0,0 +1,10 @@
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
diff --git a/test/queries_generator/aws_results/output3.csv b/test/queries_generator/aws_results/output3.csv
@@ -0,0 +1,10 @@
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
diff --git a/test/queries_generator/aws_results/output4.csv b/test/queries_generator/aws_results/output4.csv
@@ -0,0 +1 @@
+
diff --git a/test/queries_generator/aws_results/output5.csv b/test/queries_generator/aws_results/output5.csv
@@ -0,0 +1,10 @@
+-42916
+-21159
+-35571
+-38378
+-42792
+-45572
+-8538
+-22623
+-38429
+-6601
diff --git a/test/queries_generator/aws_results/output6.csv b/test/queries_generator/aws_results/output6.csv
@@ -0,0 +1,10 @@
+9
+9
+9
+9
+9
+9
+9
+9
+9
+9
diff --git a/test/queries_generator/aws_results/output7.csv b/test/queries_generator/aws_results/output7.csv
@@ -0,0 +1,10 @@
+42924
+21167
+35579
+38386
+42800
+45580
+8546
+22631
+38437
+6609
diff --git a/test/queries_generator/aws_results/output8.csv b/test/queries_generator/aws_results/output8.csv
@@ -0,0 +1,10 @@
+34
+34
+34
+34
+34
+34
+34
+34
+34
+34
diff --git a/test/queries_generator/aws_results/output9.csv b/test/queries_generator/aws_results/output9.csv
@@ -0,0 +1,10 @@
+10 
+10 
+10 
+10 
+10 
+10 
+10 
+10 
+10 
+10 
diff --git a/test/queries_generator/generate_aws_cmds b/test/queries_generator/generate_aws_cmds
diff --git a/test/queries_generator/generate_aws_cmds.cpp b/test/queries_generator/generate_aws_cmds.cpp
@@ -0,0 +1,26 @@
+#include <iostream>
+#include <fstream>
+
+using namespace std;
+
+int main()
+{
+    fstream query_file, cmd_file;
+    query_file.open("aws_queries.txt", ios::in);
+    cmd_file.open("aws_cmds.sh", ios::out);
+    cmd_file << "#!/bin/sh\nset -x\nset -e\n\n";
+    cmd_file << "mkdir -p aws_results\n";
+    string bucket, csv_file, query, aws_cmd;
+    cout << "Enter bucket name: ";
+    cin >> bucket;
+    cout << "Enter file name: ";
+    cin >> csv_file;
+    for(int i = 1; getline(query_file, query); i++)
+    {
+        aws_cmd = "aws s3api select-object-content --bucket " + bucket + " --key " + csv_file + "  --expression-type \'SQL\' --input-serialization \'{\"CSV\": {}, \"CompressionType\": \"NONE\"}\' --output-serialization \'{\"CSV\": {}}\' --profile openshift-dev --expression \"" + query + "\" \"aws_results/output" + to_string(i) + ".csv\"";
+	cmd_file << aws_cmd << endl;
+    }
+    cmd_file.close();
+    query_file.close();
+    return 0;
+}
diff --git a/test/queries_generator/queries.txt b/test/queries_generator/queries.txt
@@ -0,0 +1,10 @@
+select lower(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from stdin;
+select to_timestamp('1999-11-04T20:27:03.479340Z') from stdin;
+select date_add(day, int(8), to_timestamp('1982-02-03T00:20:12.42831Z')) from stdin;
+select substring(upper('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  '), int(avg(int(_3)-int(_2)) - int(3)), min(int(_1)-int(_1)) + int(7)) from stdin;
+select int(10)-int(_2) from stdin;
+select character_length(to_string(to_timestamp('1963-06-22T05:35:39.954350Z'), 'SSSSS HH ')) from stdin;
+select int(_2)-int(2) from stdin;
+select char_length(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from stdin;
+select to_string(date_add(second, int(5), to_timestamp('1938-09-24T10:57:42.142042Z')), 'hh ') from stdin;
+select to_timestamp('1931-12-23T02:54:25.451925Z') from stdin;