Merge pull request #59 from Matts966/feature/52-with-tables-and-funct…

…ions

Add options --with_tables and --with_functions

Makefile

@@ -12,6 +12,18 @@ osx:
 sample: osx
 	ls -d samples/*/ | while read sample_path; do \
 		echo ""; \
+		alphadag --with_functions $$sample_path --output_path $$sample_path/with_funtcions/dag.dot \
+			--external_required_tables_output_path $$sample_path/with_funtcions/external_tables.txt \
+		 	> $$sample_path/with_funtcions/alphadag_stdout.txt 2> $$sample_path/with_funtcions/alphadag_stderr.txt; \
+		dot -Tpng $$sample_path/with_funtcions/dag.dot -o $$sample_path/with_funtcions/dag.png; \
+		alphadag --with_tables $$sample_path --output_path $$sample_path/with_tables/dag.dot \
+			--external_required_tables_output_path $$sample_path/with_tables/external_tables.txt \
+		 	> $$sample_path/with_tables/alphadag_stdout.txt 2> $$sample_path/with_tables/alphadag_stderr.txt; \
+		dot -Tpng $$sample_path/with_tables/dag.dot -o $$sample_path/with_tables/dag.png; \
+		alphadag --with_tables --with_functions $$sample_path --output_path $$sample_path/with_all/dag.dot \
+			--external_required_tables_output_path $$sample_path/with_all/external_tables.txt \
+		 	> $$sample_path/with_all/alphadag_stdout.txt 2> $$sample_path/with_all/alphadag_stderr.txt; \
+		dot -Tpng $$sample_path/with_all/dag.dot -o $$sample_path/with_all/dag.png; \
 		alphadag $$sample_path --output_path $$sample_path/dag.dot \
 			--external_required_tables_output_path $$sample_path/external_tables.txt \
 		 	> $$sample_path/alphadag_stdout.txt 2> $$sample_path/alphadag_stderr.txt; \

alphasql/alphadag.cc

@@ -21,6 +21,12 @@
 
 std::regex DEFAULT_EXCLUDES(".*(.git/.*|.hg/.*|.svn/.*)");
 
+ABSL_FLAG(bool, with_tables, false,
+          "Show DAG with tables.");
+
+ABSL_FLAG(bool, with_functions, false,
+          "Show DAG with functions.");
+
 int main(int argc, char* argv[]) {
   const char kUsage[] =
       "Usage: alphadag --external_required_tables_output_path <filename> --output_path <filename> <directory or file paths of sql...>\n";
@@ -74,29 +80,65 @@ int main(int argc, char* argv[]) {
 
   std::vector<Edge> depends_on;
   std::vector<std::string> external_required_tables;
+  std::set<std::string> table_vertices;
+  const bool with_tables = absl::GetFlag(FLAGS_with_tables);
   for (auto const& [table_name, table_queries] : table_queries_map) {
-    alphasql::UpdateEdges(depends_on, table_queries.others, table_queries.create);
+    if (with_tables) {
+      alphasql::UpdateEdges(depends_on, table_queries.others, table_name);
+      alphasql::UpdateEdges(depends_on, {table_name}, table_queries.create);
+      table_vertices.insert(table_name);
+    } else {
+      alphasql::UpdateEdges(depends_on, table_queries.others, table_queries.create);
+    }
     if (table_queries.create.empty()) {
       external_required_tables.push_back(table_name);
     }
   }
 
-  for (auto const& [_, function_queries] : function_queries_map) {
-    alphasql::UpdateEdges(depends_on, function_queries.call, function_queries.create);
+  const bool with_functions = absl::GetFlag(FLAGS_with_functions);
+  std::set<std::string> function_vertices;
+  for (auto const& [function_name, function_queries] : function_queries_map) {
+    if (with_functions && !function_queries.create.empty()) { // Skip default functions
+      alphasql::UpdateEdges(depends_on, function_queries.call, function_name);
+      alphasql::UpdateEdges(depends_on, {function_name}, function_queries.create);
+      function_vertices.insert(function_name);
+    } else {
+      alphasql::UpdateEdges(depends_on, function_queries.call, function_queries.create);
+    }
   }
 
   const int nedges = depends_on.size();
 
   using namespace boost;
 
-  typedef adjacency_list<vecS, vecS, directedS, property<vertex_name_t, std::string>> Graph;
-  Graph g(vertices.size());
+  struct vertex_info_t { 
+    std::string label; 
+    std::string shape;
+    std::string type;
+  };
+  typedef adjacency_list<vecS, vecS, directedS, vertex_info_t> Graph;
+  Graph g(vertices.size() + table_vertices.size() + function_vertices.size());
 
   std::map<std::string, Graph::vertex_descriptor> indexes;
   // fills the property 'vertex_name_t' of the vertices
   int i = 0;
   for (const auto& vertice : vertices) {
-    put(vertex_name_t(), g, i, vertice); // set the property of a vertex
+    g[i].label = vertice; // set the property of a vertex
+    g[i].type = "query";
+    indexes[vertice] = vertex(i, g);     // retrives the associated vertex descriptor
+    ++i;
+  }
+  for (const auto& vertice : table_vertices) {
+    g[i].label = vertice; // set the property of a vertex
+    g[i].type = "table";
+    g[i].shape = "box";
+    indexes[vertice] = vertex(i, g);     // retrives the associated vertex descriptor
+    ++i;
+  }
+  for (const auto& vertice : function_vertices) {
+    g[i].label = vertice; // set the property of a vertex
+    g[i].type = "function";
+    g[i].shape = "cds";
     indexes[vertice] = vertex(i, g);     // retrives the associated vertex descriptor
     ++i;
   }
@@ -110,17 +152,22 @@ int main(int argc, char* argv[]) {
     add_edge(indexes[depends_on[i].second], indexes[depends_on[i].first], g);
   }
 
+  boost::dynamic_properties dp;
+  dp.property("shape", get(&vertex_info_t::shape, g));
+  dp.property("type", get(&vertex_info_t::type, g));
+  dp.property("label", get(&vertex_info_t::label, g));
+  dp.property("node_id", get(boost::vertex_index, g));
   const std::string output_path = absl::GetFlag(FLAGS_output_path);
   if (output_path.empty()) {
-    write_graphviz(std::cout, g, make_label_writer(get(vertex_name, g)));
+    write_graphviz_dp(std::cout, g, dp);
   } else {
     if (std::filesystem::is_regular_file(output_path) || std::filesystem::is_fifo(output_path) || !std::filesystem::exists(output_path)) {
       std::filesystem::path parent = std::filesystem::path(output_path).parent_path();
       if (!std::filesystem::is_directory(parent)) {
         std::filesystem::create_directories(parent);
       }
       std::ofstream out(output_path);
-      write_graphviz(out, g, make_label_writer(get(vertex_name, g)));
+      write_graphviz_dp(out, g, dp);
     } else {
       std::cout << "output_path is not a file!" << std::endl;
       return 1;

alphasql/dag_lib.h

@@ -87,11 +87,17 @@ namespace alphasql {
     // Currently resolve drop statements as reference.
     for (auto const& table_name : identifier_information.table_information.dropped) {
       const std::string table_string = absl::StrJoin(table_name, ".");
+      if (table_queries_map[table_string].create == file_path) {
+        continue;
+      }
       table_queries_map[table_string].others.push_back(file_path);
     }
 
     for (auto const& table_name : identifier_information.table_information.referenced) {
       const std::string table_string = absl::StrJoin(table_name, ".");
+      if (table_queries_map[table_string].create == file_path) {
+        continue;
+      }
       table_queries_map[table_string].others.push_back(file_path);
     }
 
@@ -115,12 +121,18 @@ namespace alphasql {
     // Currently resolve drop statements as reference.
     for (auto const& dropped : identifier_information.function_information.dropped) {
       const std::string function_name = absl::StrJoin(dropped, ".");
+      if (function_queries_map[function_name].create == file_path) {
+        continue;
+      }
       function_queries_map[function_name].call.push_back(file_path);
     }
 
 
     for (auto const& called : identifier_information.function_information.called) {
       const std::string function_name = absl::StrJoin(called, ".");
+      if (function_queries_map[function_name].create == file_path) {
+        continue;
+      }
       function_queries_map[function_name].call.push_back(file_path);
     }
 
@@ -134,9 +146,7 @@ namespace alphasql {
                    std::vector<std::string> dependents, std::string parent) {
     if (!dependents.size() || parent.empty()) return;
     for (const std::string& dep : dependents) {
-      if (dep != parent) {
-        depends_on.push_back(std::make_pair(dep, parent));
-      }
+      depends_on.push_back(std::make_pair(dep, parent));
     }
   }
 }

samples/create-temp-table-test/dag.dot

@@ -1,4 +1,4 @@
 digraph G {
-0[label="samples/create-temp-table-test/reference.sql"];
-1[label="samples/create-temp-table-test/temp_table.sql"];
+0 [label="samples/create-temp-table-test/reference.sql", shape="", type=query];
+1 [label="samples/create-temp-table-test/temp_table.sql", shape="", type=query];
 }

samples/create-temp-table-test/with_all/alphadag_stdout.txt

@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/create-temp-table-test/temp_table.sql"
+Reading "samples/create-temp-table-test/reference.sql"

samples/create-temp-table-test/with_all/dag.dot

@@ -0,0 +1,4 @@
+digraph G {
+0 [label="samples/create-temp-table-test/reference.sql", shape="", type=query];
+1 [label="samples/create-temp-table-test/temp_table.sql", shape="", type=query];
+}

samples/create-temp-table-test/with_funtcions/alphadag_stdout.txt

@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/create-temp-table-test/temp_table.sql"
+Reading "samples/create-temp-table-test/reference.sql"

samples/create-temp-table-test/with_funtcions/dag.dot

@@ -0,0 +1,4 @@
+digraph G {
+0 [label="samples/create-temp-table-test/reference.sql", shape="", type=query];
+1 [label="samples/create-temp-table-test/temp_table.sql", shape="", type=query];
+}

samples/create-temp-table-test/with_tables/alphadag_stdout.txt

@@ -0,0 +1,4 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/create-temp-table-test/temp_table.sql"
+Reading "samples/create-temp-table-test/reference.sql"

samples/create-temp-table-test/with_tables/dag.dot

@@ -0,0 +1,4 @@
+digraph G {
+0 [label="samples/create-temp-table-test/reference.sql", shape="", type=query];
+1 [label="samples/create-temp-table-test/temp_table.sql", shape="", type=query];
+}

samples/drop-test/dag.dot

@@ -1,3 +1,3 @@
 digraph G {
-0[label="samples/drop-test/create-and-drop.sql"];
+0 [label="samples/drop-test/create-and-drop.sql", shape="", type=query];
 }

samples/drop-test/with_all/alphadag_stdout.txt

@@ -0,0 +1,3 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/drop-test/create-and-drop.sql"

samples/drop-test/with_all/dag.dot

@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/drop-test/create-and-drop.sql", shape="", type=query];
+1 [label=table, shape=box, type=table];
+0->1 ;
+}

samples/drop-test/with_funtcions/alphadag_stdout.txt

@@ -0,0 +1,3 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/drop-test/create-and-drop.sql"

samples/drop-test/with_funtcions/dag.dot

@@ -0,0 +1,3 @@
+digraph G {
+0 [label="samples/drop-test/create-and-drop.sql", shape="", type=query];
+}

samples/drop-test/with_tables/alphadag_stdout.txt

@@ -0,0 +1,3 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/drop-test/create-and-drop.sql"

samples/drop-test/with_tables/dag.dot

@@ -0,0 +1,5 @@
+digraph G {
+0 [label="samples/drop-test/create-and-drop.sql", shape="", type=query];
+1 [label=table, shape=box, type=table];
+0->1 ;
+}

samples/sample-arbitrary-dependency-graph-with-drop-statement/dag.dot

@@ -1,8 +1,8 @@
 digraph G {
-0[label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"];
-1[label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"];
-2[label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"];
-3[label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"];
+0 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql", shape="", type=query];
+1 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql", shape="", type=query];
+2 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql", shape="", type=query];
+3 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql", shape="", type=query];
 1->2 ;
 1->3 ;
 }

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_all/alphadag_stdout.txt

@@ -0,0 +1,10 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"
+Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
+This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"
+Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
+This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_all/dag.dot

@@ -0,0 +1,15 @@
+digraph G {
+0 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql", shape="", type=query];
+1 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql", shape="", type=query];
+2 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql", shape="", type=query];
+3 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql", shape="", type=query];
+4 [label="dataset.dependency", shape=box, type=table];
+5 [label="dataset.main", shape=box, type=table];
+1->4 ;
+4->2 ;
+4->3 ;
+5->2 ;
+5->3 ;
+5->0 ;
+5->1 ;
+}

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_all/external_tables.txt

@@ -0,0 +1 @@
+dataset.main

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_funtcions/alphadag_stdout.txt

@@ -0,0 +1,10 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"
+Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
+This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"
+Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
+This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_funtcions/dag.dot

@@ -0,0 +1,8 @@
+digraph G {
+0 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql", shape="", type=query];
+1 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql", shape="", type=query];
+2 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql", shape="", type=query];
+3 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql", shape="", type=query];
+1->2 ;
+1->3 ;
+}

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_funtcions/external_tables.txt

@@ -0,0 +1 @@
+dataset.main

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_tables/alphadag_stdout.txt

@@ -0,0 +1,10 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql"
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql"
+Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
+This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql"
+Warning!!! the target of INSERT statement dataset.main is not created in the same script!!!
+This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_tables/dag.dot

@@ -0,0 +1,15 @@
+digraph G {
+0 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert.sql", shape="", type=query];
+1 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/insert_before_reference.sql", shape="", type=query];
+2 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select.sql", shape="", type=query];
+3 [label="samples/sample-arbitrary-dependency-graph-with-drop-statement/select2.sql", shape="", type=query];
+4 [label="dataset.dependency", shape=box, type=table];
+5 [label="dataset.main", shape=box, type=table];
+1->4 ;
+4->2 ;
+4->3 ;
+5->2 ;
+5->3 ;
+5->0 ;
+5->1 ;
+}

samples/sample-arbitrary-dependency-graph-with-drop-statement/with_tables/external_tables.txt

@@ -0,0 +1 @@
+dataset.main

samples/sample-ci/alphacheck_stdout.txt

@@ -1,6 +1,6 @@
 Analyzing "samples/sample-ci/sample/create_datawarehouse3.sql"
 ERROR: INVALID_ARGUMENT: Table not found: `bigquery-public-data.samples.gsod` [at samples/sample-ci/sample/create_datawarehouse3.sql:5:3]
 catalog:
-	tablename1
-	tablename2
 	dataset.main
+	tablename2
+	tablename1

samples/sample-ci/dag.dot

@@ -1,13 +1,13 @@
 digraph G {
-0[label="samples/sample-ci/sample/create_datawarehouse1.sql"];
-1[label="samples/sample-ci/sample/create_datawarehouse2.sql"];
-2[label="samples/sample-ci/sample/create_datawarehouse3.sql"];
-3[label="samples/sample-ci/sample/create_interim1.sql"];
-4[label="samples/sample-ci/sample/create_interim2.sql"];
-5[label="samples/sample-ci/sample/create_interim3.sql"];
-6[label="samples/sample-ci/sample/create_mart.sql"];
-7[label="samples/sample-ci/sample/insert_into_interim1.sql"];
-8[label="samples/sample-ci/sample/update_interim2.sql"];
+0 [label="samples/sample-ci/sample/create_datawarehouse1.sql", shape="", type=query];
+1 [label="samples/sample-ci/sample/create_datawarehouse2.sql", shape="", type=query];
+2 [label="samples/sample-ci/sample/create_datawarehouse3.sql", shape="", type=query];
+3 [label="samples/sample-ci/sample/create_interim1.sql", shape="", type=query];
+4 [label="samples/sample-ci/sample/create_interim2.sql", shape="", type=query];
+5 [label="samples/sample-ci/sample/create_interim3.sql", shape="", type=query];
+6 [label="samples/sample-ci/sample/create_mart.sql", shape="", type=query];
+7 [label="samples/sample-ci/sample/insert_into_interim1.sql", shape="", type=query];
+8 [label="samples/sample-ci/sample/update_interim2.sql", shape="", type=query];
 0->3 ;
 0->5 ;
 0->6 ;

samples/sample-ci/with_all/alphadag_stdout.txt

@@ -0,0 +1,15 @@
+Reading paths passed as a command line arguments...
+Only files that end with .sql or .bq are analyzed.
+Reading "samples/sample-ci/sample/create_interim1.sql"
+Reading "samples/sample-ci/sample/create_interim3.sql"
+Reading "samples/sample-ci/sample/create_interim2.sql"
+Reading "samples/sample-ci/sample/update_interim2.sql"
+Warning!!! the target of UPDATE statement interim2 is not created in the same script!!!
+This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-ci/sample/insert_into_interim1.sql"
+Warning!!! the target of INSERT statement interim1 is not created in the same script!!!
+This script is not idempotent. See https://github.com/Matts966/alphasql/issues/5#issuecomment-735209829 for more details.
+Reading "samples/sample-ci/sample/create_datawarehouse1.sql"
+Reading "samples/sample-ci/sample/create_datawarehouse3.sql"
+Reading "samples/sample-ci/sample/create_datawarehouse2.sql"
+Reading "samples/sample-ci/sample/create_mart.sql"