Merge pull request #75 from Matts966/hotfix/temp-reference

Fix temporary table references

.github/workflows/test.yml

@@ -20,16 +20,19 @@ jobs:
       - name: Setup
         run: |
           sudo apt-get update
-          sudo apt-get install --no-install-recommends -y gcc-9 g++-9
+          sudo apt-get install --no-install-recommends -y gcc-9 g++-9 graphviz
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 900 \
             --slave /usr/bin/g++ g++ /usr/bin/g++-9
-          bazelisk build //alphasql/...
-          bazelisk test --test_output=errors //alphasql/...
+          make
 
-          ./bazel-bin/alphasql/alphadag ./samples/sample
-          ./bazel-bin/alphasql/alphadag --output_path=dag.dot ./samples/sample
-          ./bazel-bin/alphasql/alphadag --external_required_tables_output_path=dag.dot ./samples/sample
+          echo 🚨 checking sample testing result...
+          echo 🚨 exit if not up-to-date
+          # TODO: check dag output isomorphism
+          git diff --exit-code -- '*.txt'
 
+          alphadag ./samples/sample
+          alphadag --output_path=dag.dot ./samples/sample
+          alphadag --external_required_tables_output_path=dag.dot ./samples/sample
 
   osx:
     name: Test the repository on Mac
@@ -50,8 +53,14 @@ jobs:
           brew install graphviz
           sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer
           bazel shutdown
+          export CC=g++
           make
 
+          echo 🚨 checking sample testing result...
+          echo 🚨 exit if not up-to-date
+          # TODO: check dag output isomorphism
+          git diff --exit-code -- '*.txt'
+
           alphadag ./samples/sample
           alphadag --output_path=dag.dot ./samples/sample
           alphadag --external_required_tables_output_path=dag.dot ./samples/sample

Makefile

@@ -1,23 +1,21 @@
 .PHONY: build-and-check
-build-and-check: test git-clean
+build-and-check: build
+	make test
 	make samples
 
-git-clean:
-	git diff --quiet
-
-.PHONY: osx
-osx:
-	CC=g++ bazelisk build //alphasql:all
-	sudo cp ./bazel-bin/alphasql/alphadag /usr/local/bin
-	sudo chmod +x /usr/local/bin/alphadag
-	sudo cp ./bazel-bin/alphasql/alphacheck /usr/local/bin
-	sudo chmod +x /usr/local/bin/alphacheck
+.PHONY: build
+build:
+	bazel build //alphasql:all
+	cp ./bazel-bin/alphasql/alphadag /usr/local/bin
+	chmod +x /usr/local/bin/alphadag
+	cp ./bazel-bin/alphasql/alphacheck /usr/local/bin
+	chmod +x /usr/local/bin/alphacheck
 
 .PHONY: samples
 samples: without_options with_functions with_tables with_all side_effect_first side_effect_first_with_tables
 
 .PHONY: without_options
-without_options: osx
+without_options:
 	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
 		alphadag $$sample_path --output_path $$sample_path/dag.dot \
 			--external_required_tables_output_path $$sample_path/external_tables.txt \
@@ -29,7 +27,7 @@ without_options: osx
 	done;
 
 .PHONY: with_functions
-with_functions: osx
+with_functions:
 	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
 	  mkdir -p $$sample_path/with_functions; \
 		alphadag --with_functions $$sample_path --output_path $$sample_path/with_functions/dag.dot \
@@ -39,7 +37,7 @@ with_functions: osx
 	done;
 
 .PHONY: with_tables
-with_tables: osx
+with_tables:
 	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
 	  mkdir -p $$sample_path/with_tables; \
     alphadag --with_tables $$sample_path --output_path $$sample_path/with_tables/dag.dot \
@@ -49,7 +47,7 @@ with_tables: osx
 	done;
 
 .PHONY: with_all
-with_all: osx
+with_all:
 	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
 	  mkdir -p $$sample_path/with_all; \
 		alphadag --with_tables --with_functions $$sample_path --output_path $$sample_path/with_all/dag.dot \
@@ -59,7 +57,7 @@ with_all: osx
 	done;
 
 .PHONY: side_effect_first
-side_effect_first: osx
+side_effect_first:
 	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
 	  mkdir -p $$sample_path/side_effect_first; \
 		alphadag --side_effect_first $$sample_path --output_path $$sample_path/side_effect_first/dag.dot \
@@ -68,7 +66,7 @@ side_effect_first: osx
     dot -Tpng $$sample_path/side_effect_first/dag.dot -o $$sample_path/side_effect_first/dag.png; \
 	done;
 
-side_effect_first_with_tables: osx
+side_effect_first_with_tables:
 	find samples -mindepth 1 -maxdepth 1 -type d | while read sample_path; do \
 	  mkdir -p $$sample_path/side_effect_first_with_tables; \
 	 	alphadag --side_effect_first --with_tables $$sample_path --output_path $$sample_path/side_effect_first_with_tables/dag.dot \
@@ -78,5 +76,5 @@ side_effect_first_with_tables: osx
 	done;
 
 .PHONY: test
-test: osx
-	CC=g++ bazelisk test //alphasql:all
+test:
+	bazel test //alphasql:all

alphasql/alphacheck.cc

@@ -325,6 +325,18 @@ absl::Status Run(const std::string &sql_file_path,
   return absl::OkStatus();
 }
 
+// TODO: Hide implementation and unify
+struct cycle_detector : public boost::dfs_visitor<> {
+  cycle_detector(bool &has_cycle) : _has_cycle(has_cycle) {}
+
+  template <class Edge, class Graph> void back_edge(Edge, Graph &) {
+    _has_cycle = true;
+  }
+
+protected:
+  bool &_has_cycle;
+};
+
 bool GetExecutionPlan(const std::string dot_path,
                       std::vector<std::string> &execution_plan) {
   using namespace boost;
@@ -339,6 +351,16 @@ bool GetExecutionPlan(const std::string dot_path,
   if (!boost::read_graphviz(file, g, dp)) {
     return false;
   }
+
+  bool has_cycle = false;
+  cycle_detector vis(has_cycle);
+  depth_first_search(g, visitor(vis));
+  if (has_cycle) {
+    std::cerr << "ERROR: cycle detected! [at " << dot_path << ":1:1]"
+              << std::endl;
+    exit(1);
+  }
+
   std::list<int> result;
   topological_sort(g, std::front_inserter(result));
   property_map<Graph, vertex_name_t>::type names = get(vertex_name, g);

alphasql/alphadag.cc

@@ -65,20 +65,20 @@ int main(int argc, char *argv[]) {
       }
       continue;
     }
-    std::filesystem::recursive_directory_iterator file_path(
-        path, std::filesystem::directory_options::skip_permission_denied),
-        end;
+    std::vector<std::filesystem::path> files_in_directory;
+    std::copy(std::filesystem::recursive_directory_iterator(path, std::filesystem::directory_options::skip_permission_denied), std::filesystem::recursive_directory_iterator(), std::back_inserter(files_in_directory));
+    std::sort(files_in_directory.begin(), files_in_directory.end());
     std::error_code err;
-    for (; file_path != end; file_path.increment(err)) {
-      std::string path_str = file_path->path().string();
+    for (const std::filesystem::path & file_path : files_in_directory) {
+      std::string path_str = file_path.string();
       if (regex_match(path_str, m, DEFAULT_EXCLUDES)) {
         continue;
       }
       if (err) {
         std::cout << "WARNING: " << err << std::endl;
       }
       absl::Status status = alphasql::UpdateIdentifierQueriesMapsAndVertices(
-          file_path->path(), table_queries_map, function_queries_map, vertices);
+          file_path, table_queries_map, function_queries_map, vertices);
       if (!status.ok()) {
         status = zetasql::UpdateErrorLocationPayloadWithFilenameIfNotPresent(status, path);
         std::cerr << status << std::endl;

alphasql/identifier_resolver.cc

@@ -105,6 +105,11 @@ GetIdentifierInformation(const std::string &sql_file_path) {
 void IdentifierResolver::visitASTDropStatement(const ASTDropStatement *node,
                                                void *data) {
   if (node->schema_object_kind() == SchemaObjectKind::kTable) {
+    const auto table_name = absl::StrJoin(node->name()->ToIdentifierVector(), ".");
+    if (temporary_tables.find(table_name) != temporary_tables.end()) {
+      visitASTChildren(node, data);
+      return;
+    }
     identifier_information.table_information.dropped.insert(
         node->name()->ToIdentifierVector());
   }
@@ -143,6 +148,11 @@ void IdentifierResolver::visitASTInsertStatement(const ASTInsertStatement *node,
   }
 
   const auto path = status_or_path.value()->ToIdentifierVector();
+  const auto table_name = absl::StrJoin(path, ".");
+  if (temporary_tables.find(table_name) != temporary_tables.end()) {
+    visitASTChildren(node, data);
+    return;
+  }
   identifier_information.table_information.inserted.insert(path);
 
   const std::string path_str = absl::StrJoin(path, ".");
@@ -177,6 +187,11 @@ void IdentifierResolver::visitASTUpdateStatement(const ASTUpdateStatement *node,
   }
 
   const auto path = status_or_path.value()->ToIdentifierVector();
+  const auto table_name = absl::StrJoin(path, ".");
+  if (temporary_tables.find(table_name) != temporary_tables.end()) {
+    visitASTChildren(node, data);
+    return;
+  }
   identifier_information.table_information.updated.insert(path);
 
   const std::string path_str = absl::StrJoin(path, ".");

samples/create-temp-table-test/alphadag_stdout.txt

@@ -1,4 +1,4 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/create-temp-table-test/temp_table.sql"
 Reading "samples/create-temp-table-test/reference.sql"
+Reading "samples/create-temp-table-test/temp_table.sql"

samples/create-temp-table-test/reference.sql

@@ -1,2 +1,3 @@
 CREATE TEMP TABLE tmp AS SELECT 1 as tmp_column;
 SELECT * FROM tmp;
+INSERT INTO tmp SELECT 2;

samples/create-temp-table-test/side_effect_first/alphadag_stdout.txt

@@ -1,4 +1,4 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/create-temp-table-test/temp_table.sql"
 Reading "samples/create-temp-table-test/reference.sql"
+Reading "samples/create-temp-table-test/temp_table.sql"

samples/create-temp-table-test/side_effect_first_with_tables/alphadag_stdout.txt

@@ -1,4 +1,4 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/create-temp-table-test/temp_table.sql"
 Reading "samples/create-temp-table-test/reference.sql"
+Reading "samples/create-temp-table-test/temp_table.sql"

samples/create-temp-table-test/with_all/alphadag_stdout.txt

@@ -1,4 +1,4 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/create-temp-table-test/temp_table.sql"
 Reading "samples/create-temp-table-test/reference.sql"
+Reading "samples/create-temp-table-test/temp_table.sql"

samples/create-temp-table-test/with_functions/alphadag_stdout.txt

@@ -1,4 +1,4 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/create-temp-table-test/temp_table.sql"
 Reading "samples/create-temp-table-test/reference.sql"
+Reading "samples/create-temp-table-test/temp_table.sql"

samples/create-temp-table-test/with_tables/alphadag_stdout.txt

@@ -1,4 +1,4 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/create-temp-table-test/temp_table.sql"
 Reading "samples/create-temp-table-test/reference.sql"
+Reading "samples/create-temp-table-test/temp_table.sql"

samples/mutasions-and-query/alphadag_stdout.txt

@@ -11,6 +11,6 @@ This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5
 Reading "samples/mutasions-and-query/mutation3.sql"
 Warning!!! the target of UPDATE statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/mutasions-and-query/query1.sql"
 Reading "samples/mutasions-and-query/query2.sql"
 Reading "samples/mutasions-and-query/query3.sql"
-Reading "samples/mutasions-and-query/query1.sql"

samples/mutasions-and-query/side_effect_first/alphadag_stdout.txt

@@ -11,6 +11,6 @@ This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5
 Reading "samples/mutasions-and-query/mutation3.sql"
 Warning!!! the target of UPDATE statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/mutasions-and-query/query1.sql"
 Reading "samples/mutasions-and-query/query2.sql"
 Reading "samples/mutasions-and-query/query3.sql"
-Reading "samples/mutasions-and-query/query1.sql"

samples/mutasions-and-query/side_effect_first_with_tables/alphadag_stdout.txt

@@ -11,6 +11,6 @@ This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5
 Reading "samples/mutasions-and-query/mutation3.sql"
 Warning!!! the target of UPDATE statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/mutasions-and-query/query1.sql"
 Reading "samples/mutasions-and-query/query2.sql"
 Reading "samples/mutasions-and-query/query3.sql"
-Reading "samples/mutasions-and-query/query1.sql"

samples/mutasions-and-query/with_all/alphadag_stdout.txt

@@ -11,6 +11,6 @@ This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5
 Reading "samples/mutasions-and-query/mutation3.sql"
 Warning!!! the target of UPDATE statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/mutasions-and-query/query1.sql"
 Reading "samples/mutasions-and-query/query2.sql"
 Reading "samples/mutasions-and-query/query3.sql"
-Reading "samples/mutasions-and-query/query1.sql"

samples/mutasions-and-query/with_functions/alphadag_stdout.txt

@@ -11,6 +11,6 @@ This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5
 Reading "samples/mutasions-and-query/mutation3.sql"
 Warning!!! the target of UPDATE statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/mutasions-and-query/query1.sql"
 Reading "samples/mutasions-and-query/query2.sql"
 Reading "samples/mutasions-and-query/query3.sql"
-Reading "samples/mutasions-and-query/query1.sql"

samples/mutasions-and-query/with_tables/alphadag_stdout.txt

@@ -11,6 +11,6 @@ This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5
 Reading "samples/mutasions-and-query/mutation3.sql"
 Warning!!! the target of UPDATE statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/mutasions-and-query/query1.sql"
 Reading "samples/mutasions-and-query/query2.sql"
 Reading "samples/mutasions-and-query/query3.sql"
-Reading "samples/mutasions-and-query/query1.sql"

samples/sample-arbitrary-dependency-graph-with-drop-statement/alphadag_stdout.txt

@@ -1,10 +1,10 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"

samples/sample-arbitrary-dependency-graph-with-drop-statement/side_effect_first/alphadag_stdout.txt

@@ -1,10 +1,10 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"

samples/sample-arbitrary-dependency-graph-with-drop-statement/side_effect_first_with_tables/alphadag_stdout.txt

@@ -1,10 +1,10 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_all/alphadag_stdout.txt

@@ -1,10 +1,10 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_all/dag.dot

@@ -8,8 +8,8 @@ digraph G {
 1->4 ;
 4->2 ;
 4->3 ;
-5->2 ;
-5->3 ;
 5->0 ;
 5->1 ;
+5->2 ;
+5->3 ;
 }

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_functions/alphadag_stdout.txt

@@ -1,10 +1,10 @@
 Reading paths passed as a command line arguments...
 Only files that end with .sql or .bq are analyzed.
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
-Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
 Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"
 Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
 This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"