Skip to content

Commit

Permalink
Merge pull request #70 from Matts966/feature/support-scripting
Browse files Browse the repository at this point in the history
Support for parsing scripts (only BEGIN END blocks are confirmed)
  • Loading branch information
Matts966 authored Sep 26, 2021
2 parents dec1871 + 81bbfe6 commit f6fa717
Show file tree
Hide file tree
Showing 41 changed files with 473 additions and 107 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ without_options: osx
dot -Tpng $$sample_path/dag.dot -o $$sample_path/dag.png; \
alphacheck $$sample_path/dag.dot \
--json_schema_path ./samples/sample-schema.json \
> $$sample_path/alphacheck_stdout.txt 2> $$sample_path/alphacheck_stderr.txt; \
> $$sample_path/alphacheck_stdout.txt 2> $$sample_path/alphacheck_stderr.txt || echo alphacheck $$sample_path failed; \
done;

.PHONY: with_functions
Expand Down
10 changes: 10 additions & 0 deletions alphasql/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,19 @@ cc_library(
],
)

cc_library(
name = "common_lib",
hdrs = ["common_lib.h"],
deps = [
"@com_google_zetasql//zetasql/parser:parser",
],
)

cc_library(
name = "table_name_resolver",
hdrs = ["table_name_resolver.h"],
deps = [
":common_lib",
"@com_google_zetasql//zetasql/public:simple_catalog",
"@com_google_zetasql//zetasql/public:type",
"@boost//:property_tree",
Expand All @@ -65,6 +74,7 @@ cc_binary(
],
deps = [
":json_schema_reader",
":common_lib",
"@com_google_zetasql//zetasql/base",
"@com_google_zetasql//zetasql/base:map_util",
"@com_google_zetasql//zetasql/base:ret_check",
Expand Down
246 changes: 179 additions & 67 deletions alphasql/alphacheck.cc
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "zetasql/base/logging.h"
#include "zetasql/public/analyzer.h"
#include "zetasql/public/catalog.h"
#include "zetasql/public/error_helpers.h"
#include "zetasql/public/evaluator.h"
#include "zetasql/public/evaluator_table_iterator.h"
#include "zetasql/public/language_options.h"
Expand All @@ -42,6 +43,7 @@
#include "zetasql/public/value.h"
#include "zetasql/resolved_ast/resolved_ast.h"

#include "alphasql/common_lib.h"
#include "alphasql/json_schema_reader.h"
#include "boost/graph/graphviz.hpp"
#include "zetasql/base/status.h"
Expand Down Expand Up @@ -122,6 +124,92 @@ SimpleCatalog *ConstructCatalog(const google::protobuf::DescriptorPool *pool,
return catalog;
}

absl::Status check(const std::string &sql, const ASTStatement *statement,
std::vector<std::string> *temp_function_names,
std::vector<std::string> *temp_table_names,
const AnalyzerOptions &options, SimpleCatalog *catalog) {
std::unique_ptr<const AnalyzerOutput> output;

if (statement->node_kind() == AST_BEGIN_END_BLOCK) {
const ASTBeginEndBlock *stmt = statement->GetAs<ASTBeginEndBlock>();
for (const auto &body : stmt->statement_list_node()->statement_list()) {
ZETASQL_RETURN_IF_ERROR(check(sql, body, temp_function_names,
temp_table_names, options, catalog));
}
if (stmt->handler_list() == nullptr) {
return absl::OkStatus();
}
for (const ASTExceptionHandler *handler :
stmt->handler_list()->exception_handler_list()) {
auto exception_handlers = handler->statement_list()->statement_list();
for (const auto &handler : exception_handlers) {
ZETASQL_RETURN_IF_ERROR(check(sql, handler, temp_function_names,
temp_table_names, options, catalog));
}
}
return absl::OkStatus();
}
ZETASQL_RETURN_IF_ERROR(AnalyzeStatementFromParserAST(
*statement, options, sql, catalog, catalog->type_factory(), &output));
auto resolved_statement = output->resolved_statement();
switch (resolved_statement->node_kind()) {
case RESOLVED_CREATE_TABLE_STMT:
case RESOLVED_CREATE_TABLE_AS_SELECT_STMT: {
auto *create_table_stmt =
resolved_statement->GetAs<ResolvedCreateTableStmt>();
std::cout << "DDL analyzed, adding table to catalog..." << std::endl;
std::string table_name = absl::StrJoin(create_table_stmt->name_path(), ".");
std::unique_ptr<zetasql::SimpleTable> table(
new zetasql::SimpleTable(table_name));
for (const auto &column_definition :
create_table_stmt->column_definition_list()) {
std::unique_ptr<zetasql::SimpleColumn> column(new SimpleColumn(
table_name, column_definition->column().name_id().ToString(),
catalog->type_factory()->MakeSimpleType(
column_definition->column().type()->kind())));
ZETASQL_RETURN_IF_ERROR(table->AddColumn(column.release(), false));
}
catalog->AddOwnedTable(table.release());
if (create_table_stmt->create_scope() ==
ResolvedCreateStatement::CREATE_TEMP) {
temp_table_names->push_back(table_name);
}
break;
}
case RESOLVED_CREATE_FUNCTION_STMT: {
auto *create_function_stmt =
resolved_statement->GetAs<ResolvedCreateFunctionStmt>();
std::cout
<< "Create Function Statement analyzed, adding function to catalog..."
<< std::endl;
std::string function_name =
absl::StrJoin(create_function_stmt->name_path(), ".");
Function *function = new Function(function_name, "group", Function::SCALAR);
function->AddSignature(create_function_stmt->signature());
catalog->AddOwnedFunction(function);
if (create_function_stmt->create_scope() ==
ResolvedCreateStatement::CREATE_TEMP) {
temp_function_names->push_back(function_name);
}
break;
}
case RESOLVED_DROP_STMT: {
auto *drop_stmt = resolved_statement->GetAs<ResolvedDropStmt>();
std::cout << "Drop Statement analyzed, dropping table from catalog..."
<< std::endl;
std::string table_name = absl::StrJoin(drop_stmt->name_path(), ".");
if (drop_stmt->is_if_exists()) {
zetasql::dropOwnedTableIfExists(catalog, table_name);
} else {
zetasql::dropOwnedTable(catalog, table_name);
}
break;
}
}

return absl::OkStatus();
}

// Runs the tool.
absl::Status Run(const std::string &sql_file_path,
const AnalyzerOptions &options, SimpleCatalog *catalog) {
Expand All @@ -130,76 +218,98 @@ absl::Status Run(const std::string &sql_file_path,
std::ifstream file(file_path, std::ios::in);
std::string sql(std::istreambuf_iterator<char>(file), {});

TypeFactory factory;
ParseResumeLocation location =
ParseResumeLocation::FromStringView(sql_file_path, sql);
bool at_end_of_input = false;
std::unique_ptr<const AnalyzerOutput> output;

std::vector<std::string> temp_function_names;
std::vector<std::string> temp_table_names;

while (!at_end_of_input) {
ZETASQL_RETURN_IF_ERROR(AnalyzeNextStatement(
&location, options, catalog, &factory, &output, &at_end_of_input));
auto resolved_statement = output->resolved_statement();
switch (resolved_statement->node_kind()) {
case RESOLVED_CREATE_TABLE_STMT:
case RESOLVED_CREATE_TABLE_AS_SELECT_STMT: {
auto *create_table_stmt =
resolved_statement->GetAs<ResolvedCreateTableStmt>();
std::cout << "DDL analyzed, adding table to catalog..." << std::endl;
std::string table_name =
absl::StrJoin(create_table_stmt->name_path(), ".");
std::unique_ptr<zetasql::SimpleTable> table(
new zetasql::SimpleTable(table_name));
for (const auto &column_definition :
create_table_stmt->column_definition_list()) {
std::unique_ptr<zetasql::SimpleColumn> column(new SimpleColumn(
table_name, column_definition->column().name_id().ToString(),
catalog->type_factory()->MakeSimpleType(
column_definition->column().type()->kind())));
ZETASQL_RETURN_IF_ERROR(table->AddColumn(column.release(), false));
}
catalog->AddOwnedTable(table.release());
if (create_table_stmt->create_scope() ==
ResolvedCreateStatement::CREATE_TEMP) {
temp_table_names.push_back(table_name);
}
break;
}
case RESOLVED_CREATE_FUNCTION_STMT: {
auto *create_function_stmt =
resolved_statement->GetAs<ResolvedCreateFunctionStmt>();
std::cout
<< "Create Function Statement analyzed, adding function to catalog..."
<< std::endl;
std::string function_name =
absl::StrJoin(create_function_stmt->name_path(), ".");
Function *function =
new Function(function_name, "group", Function::SCALAR);
function->AddSignature(create_function_stmt->signature());
catalog->AddOwnedFunction(function);
if (create_function_stmt->create_scope() ==
ResolvedCreateStatement::CREATE_TEMP) {
temp_function_names.push_back(function_name);
}
break;
}
case RESOLVED_DROP_STMT: {
auto *drop_stmt = resolved_statement->GetAs<ResolvedDropStmt>();
std::cout << "Drop Statement analyzed, dropping table from catalog..."
<< std::endl;
std::string table_name = absl::StrJoin(drop_stmt->name_path(), ".");
if (drop_stmt->is_if_exists()) {
zetasql::dropOwnedTableIfExists(catalog, table_name);
} else {
zetasql::dropOwnedTable(catalog, table_name);
}
break;
}
}
std::unique_ptr<ParserOutput> parser_output;
ZETASQL_RETURN_IF_ERROR(alphasql::ParseScript(sql, options.GetParserOptions(),
options.error_message_mode(),
&parser_output, file_path));

auto statements =
parser_output->script()->statement_list_node()->statement_list();
for (const ASTStatement *statement : statements) {
ZETASQL_RETURN_IF_ERROR(check(sql, statement, &temp_function_names,
&temp_table_names, options, catalog));
}
/* for (const ASTStatement *statement : statements) { */
/* if (statement->node_kind() == AST_BEGIN_END_BLOCK) { */
/* const ASTBeginEndBlock *stmt = statement->GetAs<ASTBeginEndBlock>(); */
/* auto body = stmt->statement_list_node()->statement_list(); */
/* statements.insert(statements.end(), body.begin(), body.end()); */
/* for (const ASTExceptionHandler *handler : */
/* stmt->handler_list()->exception_handler_list()) { */
/* auto exception_handlers =
* handler->statement_list()->statement_list(); */
/* statements.insert(statements.end(), exception_handlers.begin(),
* exception_handlers.end()); */
/* } */
/* continue; */
/* } */
/* ZETASQL_RETURN_IF_ERROR(AnalyzeStatementFromParserAST( */
/* *statement, options, sql, catalog, &factory, &output)); */
/* auto resolved_statement = output->resolved_statement(); */
/* switch (resolved_statement->node_kind()) { */
/* case RESOLVED_CREATE_TABLE_STMT: */
/* case RESOLVED_CREATE_TABLE_AS_SELECT_STMT: { */
/* auto *create_table_stmt = */
/* resolved_statement->GetAs<ResolvedCreateTableStmt>(); */
/* std::cout << "DDL analyzed, adding table to catalog..." << std::endl;
*/
/* std::string table_name = */
/* absl::StrJoin(create_table_stmt->name_path(), "."); */
/* std::unique_ptr<zetasql::SimpleTable> table( */
/* new zetasql::SimpleTable(table_name)); */
/* for (const auto &column_definition : */
/* create_table_stmt->column_definition_list()) { */
/* std::unique_ptr<zetasql::SimpleColumn> column(new SimpleColumn( */
/* table_name, column_definition->column().name_id().ToString(), */
/* catalog->type_factory()->MakeSimpleType( */
/* column_definition->column().type()->kind()))); */
/* ZETASQL_RETURN_IF_ERROR(table->AddColumn(column.release(), false));
*/
/* } */
/* catalog->AddOwnedTable(table.release()); */
/* if (create_table_stmt->create_scope() == */
/* ResolvedCreateStatement::CREATE_TEMP) { */
/* temp_table_names.push_back(table_name); */
/* } */
/* break; */
/* } */
/* case RESOLVED_CREATE_FUNCTION_STMT: { */
/* auto *create_function_stmt = */
/* resolved_statement->GetAs<ResolvedCreateFunctionStmt>(); */
/* std::cout */
/* << "Create Function Statement analyzed, adding function to
* catalog..." */
/* << std::endl; */
/* std::string function_name = */
/* absl::StrJoin(create_function_stmt->name_path(), "."); */
/* Function *function = */
/* new Function(function_name, "group", Function::SCALAR); */
/* function->AddSignature(create_function_stmt->signature()); */
/* catalog->AddOwnedFunction(function); */
/* if (create_function_stmt->create_scope() == */
/* ResolvedCreateStatement::CREATE_TEMP) { */
/* temp_function_names.push_back(function_name); */
/* } */
/* break; */
/* } */
/* case RESOLVED_DROP_STMT: { */
/* auto *drop_stmt = resolved_statement->GetAs<ResolvedDropStmt>(); */
/* std::cout << "Drop Statement analyzed, dropping table from catalog..."
*/
/* << std::endl; */
/* std::string table_name = absl::StrJoin(drop_stmt->name_path(), "."); */
/* if (drop_stmt->is_if_exists()) { */
/* zetasql::dropOwnedTableIfExists(catalog, table_name); */
/* } else { */
/* zetasql::dropOwnedTable(catalog, table_name); */
/* } */
/* break; */
/* } */
/* } */
/* } */

for (const auto &table_name : temp_table_names) {
std::cout << "Removing temporary table " << table_name << std::endl;
Expand Down Expand Up @@ -279,10 +389,12 @@ int main(int argc, char *argv[]) {
std::cout << "ERROR: not a file " << sql_file_path << std::endl;
return 1;
}
const absl::Status status = alphasql::Run(sql_file_path, options, catalog);
absl::Status status = alphasql::Run(sql_file_path, options, catalog);
if (status.ok()) {
std::cout << "SUCCESS: analysis finished!" << std::endl;
} else {
status = zetasql::UpdateErrorLocationPayloadWithFilenameIfNotPresent(
status, sql_file_path);
std::cout << "ERROR: " << status << std::endl;
std::cout << "catalog:" << std::endl;
for (const std::string &table_name : catalog->table_names()) {
Expand Down
61 changes: 61 additions & 0 deletions alphasql/common_lib.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
//
// Copyright 2020 Matts966
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

#include "zetasql/parser/bison_parser.h"
#include "zetasql/parser/bison_parser_mode.h"
#include "zetasql/parser/parse_tree.h"
#include "zetasql/parser/parser.h"

namespace alphasql {

using namespace zetasql;
using namespace zetasql::parser;
using zetasql::parser::BisonParser;
using zetasql::parser::BisonParserMode;

absl::Status ParseScript(absl::string_view script_string,
const ParserOptions &parser_options_in,
ErrorMessageMode error_message_mode,
std::unique_ptr<ParserOutput> *output,
const std::string &filename) {
ParserOptions parser_options = parser_options_in;
parser_options.CreateDefaultArenasIfNotSet();

BisonParser parser;
std::unique_ptr<ASTNode> ast_node;
std::vector<std::unique_ptr<ASTNode>> other_allocated_ast_nodes;
absl::Status status = parser.Parse(
BisonParserMode::kScript, filename, script_string,
/*start_byte_offset=*/0, parser_options.id_string_pool().get(),
parser_options.arena().get(), parser_options.language_options(),
&ast_node, &other_allocated_ast_nodes,
/*ast_statement_properties=*/nullptr,
/*statement_end_byte_offset=*/nullptr);

std::unique_ptr<ASTScript> script;
if (status.ok()) {
ZETASQL_RET_CHECK_EQ(ast_node->node_kind(), AST_SCRIPT);
script = absl::WrapUnique(ast_node.release()->GetAsOrDie<ASTScript>());
}
ZETASQL_RETURN_IF_ERROR(ConvertInternalErrorLocationAndAdjustErrorString(
error_message_mode, script_string, status));
*output = absl::make_unique<ParserOutput>(
parser_options.id_string_pool(), parser_options.arena(),
std::move(other_allocated_ast_nodes), std::move(script));
return absl::OkStatus();
}

} // namespace alphasql
Loading

0 comments on commit f6fa717

Please sign in to comment.