Merge pull request #76 from Matts966/feature/templated-functions

Add AlphaCheck support for templated functions
Matts966 · Jan 16, 2022 · e80ff48 · e80ff48
2 parents f5f0779 + e6e07a1
commit e80ff48
Show file tree

Hide file tree

Showing 70 changed files with 225 additions and 3 deletions.
diff --git a/alphasql/alphacheck.cc b/alphasql/alphacheck.cc
@@ -42,6 +42,7 @@
 #include "zetasql/public/simple_catalog.h"
 #include "zetasql/public/type.h"
 #include "zetasql/public/value.h"
+#include "zetasql/public/templated_sql_function.h"
 #include "zetasql/resolved_ast/resolved_ast.h"
 
 #include "alphasql/common_lib.h"
@@ -185,9 +186,19 @@ absl::Status check(const std::string &sql, const ASTStatement *statement,
         << std::endl;
     std::string function_name =
         absl::StrJoin(create_function_stmt->name_path(), ".");
-    Function *function = new Function(function_name, "group", Function::SCALAR);
-    function->AddSignature(create_function_stmt->signature());
-    catalog->AddOwnedFunction(function);
+    if (create_function_stmt->signature().IsTemplated()) {
+      TemplatedSQLFunction *function;
+      function = new TemplatedSQLFunction(
+        create_function_stmt->name_path(),
+        create_function_stmt->signature(),
+        create_function_stmt->argument_name_list(),
+        ParseResumeLocation::FromString(create_function_stmt->code()));
+      catalog->AddOwnedFunction(function);
+    } else {
+      Function *function = new Function(function_name, "group", Function::SCALAR);
+      function->AddSignature(create_function_stmt->signature());
+      catalog->AddOwnedFunction(function);
+    }
     if (create_function_stmt->create_scope() ==
         ResolvedCreateStatement::CREATE_TEMP) {
       temp_function_names->push_back(function_name);

diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile
@@ -0,0 +1,16 @@
+# syntax = docker/dockerfile:experimental
+
+FROM l.gcr.io/google/bazel:1.0.0 as builder
+
+# Use gcc because clang can't build m4
+RUN apt-get update && \
+    apt-get install build-essential software-properties-common -y && \
+    add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
+    apt-get update && \
+    # Use gcc-9 for using std::filesystem api
+    apt-get install --no-install-recommends -y make gcc-9 g++-9 graphviz tzdata && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 900 \
+        --slave /usr/bin/g++ g++ /usr/bin/g++-9
+ENV CC /usr/bin/gcc
+COPY . /work/alphasql
+WORKDIR /work/alphasql
diff --git a/samples/ml/alphacheck_stderr.txt b/samples/ml/alphacheck_stderr.txt
@@ -0,0 +1 @@
+ERROR: INVALID_ARGUMENT: Table not found: datawarehouse2 [at samples/ml/create_input.sql:5:3]
diff --git a/samples/ml/alphacheck_stdout.txt b/samples/ml/alphacheck_stdout.txt
@@ -0,0 +1,5 @@
+Analyzing "samples/ml/create_input.sql"
+catalog:
+	dataset.main
+	tablename1
+	tablename2
diff --git a/samples/ml/alphadag_stderr.txt b/samples/ml/alphadag_stderr.txt
diff --git a/samples/ml/alphadag_stdout.txt b/samples/ml/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/ml/create_input.sql"
+Reading "samples/ml/create_model.sql"
diff --git a/samples/ml/create_input.sql b/samples/ml/create_input.sql
@@ -0,0 +1,10 @@
+CREATE OR REPLACE TABLE input AS
+SELECT
+  x
+FROM
+  datawarehouse2
+UNION ALL
+SELECT
+  x
+FROM
+  datawarehouse3;
diff --git a/samples/ml/create_model.sql b/samples/ml/create_model.sql
@@ -0,0 +1,7 @@
+CREATE OR REPLACE MODEL `tmp.ml_sample`
+OPTIONS
+  (model_type='linear_reg', input_label_cols=['label']) AS
+SELECT
+      *
+FROM
+  `input`;
diff --git a/samples/ml/dag.dot b/samples/ml/dag.dot
@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/ml/create_input.sql", shape="", type=query];
+1 [label="samples/ml/create_model.sql", shape="", type=query];
+0->1 ;
+}
diff --git a/samples/ml/dag.png b/samples/ml/dag.png
diff --git a/samples/ml/external_tables.txt b/samples/ml/external_tables.txt
@@ -0,0 +1,2 @@
+datawarehouse2
+datawarehouse3
diff --git a/samples/ml/side_effect_first/alphadag_stderr.txt b/samples/ml/side_effect_first/alphadag_stderr.txt
diff --git a/samples/ml/side_effect_first/alphadag_stdout.txt b/samples/ml/side_effect_first/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/ml/create_input.sql"
+Reading "samples/ml/create_model.sql"
diff --git a/samples/ml/side_effect_first/dag.dot b/samples/ml/side_effect_first/dag.dot
@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/ml/create_input.sql", shape="", type=query];
+1 [label="samples/ml/create_model.sql", shape="", type=query];
+0->1 ;
+}
diff --git a/samples/ml/side_effect_first/dag.png b/samples/ml/side_effect_first/dag.png
diff --git a/samples/ml/side_effect_first/external_tables.txt b/samples/ml/side_effect_first/external_tables.txt
@@ -0,0 +1,2 @@
+datawarehouse2
+datawarehouse3
diff --git a/samples/ml/side_effect_first_with_tables/alphadag_stderr.txt b/samples/ml/side_effect_first_with_tables/alphadag_stderr.txt
diff --git a/samples/ml/side_effect_first_with_tables/alphadag_stdout.txt b/samples/ml/side_effect_first_with_tables/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/ml/create_input.sql"
+Reading "samples/ml/create_model.sql"
diff --git a/samples/ml/side_effect_first_with_tables/dag.dot b/samples/ml/side_effect_first_with_tables/dag.dot
@@ -0,0 +1,11 @@
+digraph G {
+0 [label="samples/ml/create_input.sql", shape="", type=query];
+1 [label="samples/ml/create_model.sql", shape="", type=query];
+2 [label=datawarehouse2, shape=box, type=table];
+3 [label=datawarehouse3, shape=box, type=table];
+4 [label=input, shape=box, type=table];
+0->4 ;
+2->0 ;
+3->0 ;
+4->1 ;
+}
diff --git a/samples/ml/side_effect_first_with_tables/dag.png b/samples/ml/side_effect_first_with_tables/dag.png
diff --git a/samples/ml/side_effect_first_with_tables/external_tables.txt b/samples/ml/side_effect_first_with_tables/external_tables.txt
@@ -0,0 +1,2 @@
+datawarehouse2
+datawarehouse3
diff --git a/samples/ml/with_all/alphadag_stderr.txt b/samples/ml/with_all/alphadag_stderr.txt
diff --git a/samples/ml/with_all/alphadag_stdout.txt b/samples/ml/with_all/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/ml/create_input.sql"
+Reading "samples/ml/create_model.sql"
diff --git a/samples/ml/with_all/dag.dot b/samples/ml/with_all/dag.dot
@@ -0,0 +1,11 @@
+digraph G {
+0 [label="samples/ml/create_input.sql", shape="", type=query];
+1 [label="samples/ml/create_model.sql", shape="", type=query];
+2 [label=datawarehouse2, shape=box, type=table];
+3 [label=datawarehouse3, shape=box, type=table];
+4 [label=input, shape=box, type=table];
+0->4 ;
+2->0 ;
+3->0 ;
+4->1 ;
+}
diff --git a/samples/ml/with_all/dag.png b/samples/ml/with_all/dag.png
diff --git a/samples/ml/with_all/external_tables.txt b/samples/ml/with_all/external_tables.txt
@@ -0,0 +1,2 @@
+datawarehouse2
+datawarehouse3
diff --git a/samples/ml/with_functions/alphadag_stderr.txt b/samples/ml/with_functions/alphadag_stderr.txt
diff --git a/samples/ml/with_functions/alphadag_stdout.txt b/samples/ml/with_functions/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/ml/create_input.sql"
+Reading "samples/ml/create_model.sql"
diff --git a/samples/ml/with_functions/dag.dot b/samples/ml/with_functions/dag.dot
@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/ml/create_input.sql", shape="", type=query];
+1 [label="samples/ml/create_model.sql", shape="", type=query];
+0->1 ;
+}
diff --git a/samples/ml/with_functions/dag.png b/samples/ml/with_functions/dag.png
diff --git a/samples/ml/with_functions/external_tables.txt b/samples/ml/with_functions/external_tables.txt
@@ -0,0 +1,2 @@
+datawarehouse2
+datawarehouse3
diff --git a/samples/ml/with_tables/alphadag_stderr.txt b/samples/ml/with_tables/alphadag_stderr.txt
diff --git a/samples/ml/with_tables/alphadag_stdout.txt b/samples/ml/with_tables/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/ml/create_input.sql"
+Reading "samples/ml/create_model.sql"
diff --git a/samples/ml/with_tables/dag.dot b/samples/ml/with_tables/dag.dot
@@ -0,0 +1,11 @@
+digraph G {
+0 [label="samples/ml/create_input.sql", shape="", type=query];
+1 [label="samples/ml/create_model.sql", shape="", type=query];
+2 [label=datawarehouse2, shape=box, type=table];
+3 [label=datawarehouse3, shape=box, type=table];
+4 [label=input, shape=box, type=table];
+0->4 ;
+2->0 ;
+3->0 ;
+4->1 ;
+}
diff --git a/samples/ml/with_tables/dag.png b/samples/ml/with_tables/dag.png
diff --git a/samples/ml/with_tables/external_tables.txt b/samples/ml/with_tables/external_tables.txt
@@ -0,0 +1,2 @@
+datawarehouse2
+datawarehouse3
diff --git a/samples/sample-any-type/alphacheck_stderr.txt b/samples/sample-any-type/alphacheck_stderr.txt
diff --git a/samples/sample-any-type/alphacheck_stdout.txt b/samples/sample-any-type/alphacheck_stdout.txt
@@ -0,0 +1,7 @@
+Analyzing "samples/sample-any-type/dedup.sql"
+Create Function Statement analyzed, adding function to catalog...
+SUCCESS: analysis finished!
+Analyzing "samples/sample-any-type/dedup_count.sql"
+Create Function Statement analyzed, adding function to catalog...
+SUCCESS: analysis finished!
+Successfully finished type check!
diff --git a/samples/sample-any-type/alphadag_stderr.txt b/samples/sample-any-type/alphadag_stderr.txt
diff --git a/samples/sample-any-type/alphadag_stdout.txt b/samples/sample-any-type/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-any-type/dedup.sql"
+Reading "samples/sample-any-type/dedup_count.sql"
diff --git a/samples/sample-any-type/dag.dot b/samples/sample-any-type/dag.dot
@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/sample-any-type/dedup.sql", shape="", type=query];
+1 [label="samples/sample-any-type/dedup_count.sql", shape="", type=query];
+0->1 ;
+}
diff --git a/samples/sample-any-type/dag.png b/samples/sample-any-type/dag.png
diff --git a/samples/sample-any-type/dedup.sql b/samples/sample-any-type/dedup.sql
@@ -0,0 +1,10 @@
+CREATE OR REPLACE FUNCTION dedup (arr ANY TYPE)
+AS ((
+  SELECT IFNULL(ARRAY_AGG(DISTINCT x), [])
+  FROM UNNEST(arr) x
+));
+ASSERT ARRAY_LENGTH(dedup([])) = 0;
+ASSERT ARRAY_LENGTH(dedup([1])) = 1;
+ASSERT ARRAY_LENGTH(dedup([1, 2])) = 2;
+ASSERT ARRAY_LENGTH(dedup([1, 3, 5])) = 3;
+ASSERT ARRAY_LENGTH(dedup([1, 3, 3, 5, 5])) = 3;
diff --git a/samples/sample-any-type/dedup_count.sql b/samples/sample-any-type/dedup_count.sql
@@ -0,0 +1,9 @@
+CREATE OR REPLACE FUNCTION dedup_count(arr ANY TYPE)
+AS (
+  ARRAY_LENGTH(dedup(arr))
+);
+ASSERT dedup_count(ARRAY[]) = 0;
+ASSERT dedup_count(ARRAY[1]) = 1;
+ASSERT dedup_count(ARRAY[1, 2]) = 2;
+ASSERT dedup_count(ARRAY[1, 3, 5]) = 3;
+ASSERT dedup_count(ARRAY[1, 3, 3, 5, 5]) = 3;
diff --git a/samples/sample-any-type/external_tables.txt b/samples/sample-any-type/external_tables.txt
diff --git a/samples/sample-any-type/side_effect_first/alphadag_stderr.txt b/samples/sample-any-type/side_effect_first/alphadag_stderr.txt
diff --git a/samples/sample-any-type/side_effect_first/alphadag_stdout.txt b/samples/sample-any-type/side_effect_first/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-any-type/dedup.sql"
+Reading "samples/sample-any-type/dedup_count.sql"
diff --git a/samples/sample-any-type/side_effect_first/dag.dot b/samples/sample-any-type/side_effect_first/dag.dot
@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/sample-any-type/dedup.sql", shape="", type=query];
+1 [label="samples/sample-any-type/dedup_count.sql", shape="", type=query];
+0->1 ;
+}
diff --git a/samples/sample-any-type/side_effect_first/dag.png b/samples/sample-any-type/side_effect_first/dag.png
diff --git a/samples/sample-any-type/side_effect_first/external_tables.txt b/samples/sample-any-type/side_effect_first/external_tables.txt
diff --git a/samples/sample-any-type/side_effect_first_with_tables/alphadag_stderr.txt b/samples/sample-any-type/side_effect_first_with_tables/alphadag_stderr.txt
diff --git a/samples/sample-any-type/side_effect_first_with_tables/alphadag_stdout.txt b/samples/sample-any-type/side_effect_first_with_tables/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-any-type/dedup.sql"
+Reading "samples/sample-any-type/dedup_count.sql"
diff --git a/samples/sample-any-type/side_effect_first_with_tables/dag.dot b/samples/sample-any-type/side_effect_first_with_tables/dag.dot
@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/sample-any-type/dedup.sql", shape="", type=query];
+1 [label="samples/sample-any-type/dedup_count.sql", shape="", type=query];
+0->1 ;
+}
diff --git a/samples/sample-any-type/side_effect_first_with_tables/dag.png b/samples/sample-any-type/side_effect_first_with_tables/dag.png
diff --git a/samples/sample-any-type/side_effect_first_with_tables/external_tables.txt b/samples/sample-any-type/side_effect_first_with_tables/external_tables.txt
diff --git a/samples/sample-any-type/with_all/alphadag_stderr.txt b/samples/sample-any-type/with_all/alphadag_stderr.txt
diff --git a/samples/sample-any-type/with_all/alphadag_stdout.txt b/samples/sample-any-type/with_all/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-any-type/dedup.sql"
+Reading "samples/sample-any-type/dedup_count.sql"
diff --git a/samples/sample-any-type/with_all/dag.dot b/samples/sample-any-type/with_all/dag.dot
@@ -0,0 +1,9 @@
+digraph G {
+0 [label="samples/sample-any-type/dedup.sql", shape="", type=query];
+1 [label="samples/sample-any-type/dedup_count.sql", shape="", type=query];
+2 [label=dedup, shape=cds, type=function];
+3 [label=dedup_count, shape=cds, type=function];
+0->2 ;
+1->3 ;
+2->1 ;
+}
diff --git a/samples/sample-any-type/with_all/dag.png b/samples/sample-any-type/with_all/dag.png
diff --git a/samples/sample-any-type/with_all/external_tables.txt b/samples/sample-any-type/with_all/external_tables.txt
diff --git a/samples/sample-any-type/with_functions/alphadag_stderr.txt b/samples/sample-any-type/with_functions/alphadag_stderr.txt
diff --git a/samples/sample-any-type/with_functions/alphadag_stdout.txt b/samples/sample-any-type/with_functions/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-any-type/dedup.sql"
+Reading "samples/sample-any-type/dedup_count.sql"
diff --git a/samples/sample-any-type/with_functions/dag.dot b/samples/sample-any-type/with_functions/dag.dot
@@ -0,0 +1,9 @@
+digraph G {
+0 [label="samples/sample-any-type/dedup.sql", shape="", type=query];
+1 [label="samples/sample-any-type/dedup_count.sql", shape="", type=query];
+2 [label=dedup, shape=cds, type=function];
+3 [label=dedup_count, shape=cds, type=function];
+0->2 ;
+1->3 ;
+2->1 ;
+}
diff --git a/samples/sample-any-type/with_functions/dag.png b/samples/sample-any-type/with_functions/dag.png
diff --git a/samples/sample-any-type/with_functions/external_tables.txt b/samples/sample-any-type/with_functions/external_tables.txt
diff --git a/samples/sample-any-type/with_tables/alphadag_stderr.txt b/samples/sample-any-type/with_tables/alphadag_stderr.txt
diff --git a/samples/sample-any-type/with_tables/alphadag_stdout.txt b/samples/sample-any-type/with_tables/alphadag_stdout.txt
@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-any-type/dedup.sql"
+Reading "samples/sample-any-type/dedup_count.sql"
diff --git a/samples/sample-any-type/with_tables/dag.dot b/samples/sample-any-type/with_tables/dag.dot
@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/sample-any-type/dedup.sql", shape="", type=query];
+1 [label="samples/sample-any-type/dedup_count.sql", shape="", type=query];
+0->1 ;
+}
diff --git a/samples/sample-any-type/with_tables/dag.png b/samples/sample-any-type/with_tables/dag.png
diff --git a/samples/sample-any-type/with_tables/external_tables.txt b/samples/sample-any-type/with_tables/external_tables.txt
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ERROR: INVALID_ARGUMENT: Table not found: datawarehouse2 [at samples/ml/create_input.sql:5:3]