Add new impl

.bazelrc

@@ -20,6 +20,9 @@
 # widely accepted by compilers. This may lead to strange behavior or compiler
 # errors in earlier compilers.
 build --cxxopt="-std=c++1z"
+build --sandbox_debug --verbose_failures
+build --cxxopt="-DNDEBUG"
+build --workspace_status_command "tools/workspace_status.sh"
 # By default, we don't suppress any warnings, to get clang-specific warning
 # suppression you can invoke with --config=clang
 build:clang --cxxopt=-Wno-deprecated-declarations

.bazelversion

@@ -1 +1 @@
-4.0.0
+4.1.0

.github/workflows/build.yaml

@@ -0,0 +1,76 @@
+name: Build
+on: push
+env:
+  cache-version: v1.0.2
+jobs:
+  linux:
+    name: Test the repository on Linux
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0
+      - name: Cache
+        uses: pat-s/[email protected]
+        with:
+          path: ~/.cache/bazel
+          key: ${{ env.cache-version }}-${{ runner.os }}-bazelisk-build-${{ hashFiles('./**') }}
+          restore-keys: ${{ env.cache-version }}-${{ runner.os }}-bazelisk-build-
+      - name: Setup
+        run: |
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends -y gcc-9 g++-9
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 900 \
+            --slave /usr/bin/g++ g++ /usr/bin/g++-9
+          bazelisk test --test_output=errors //zetasql/public:sql_formatter_test
+          bazelisk build //zetasql/tools/zetasql-formatter:format
+          sudo cp bazel-bin/zetasql/tools/zetasql-formatter/format zetasql-formatter
+          zip zetasql-formatter_linux_x86_64.zip zetasql-formatter
+      - name: Test
+        run: |
+          cd zetasql/tools/zetasql-formatter
+          ls example_tests | xargs -n1 -I {} sh -c 'cat example_tests/{} | ../../../zetasql-formatter > example_tests_formatted/{}'
+          git diff --exit-code -- '*.sql'
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: zetasql-formatter_linux_x86_64.zip
+          prerelease: true
+          generate_release_notes: true
+  macos:
+    name: Test the repository
+    runs-on: macos-10.15
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0
+      - name: Cache
+        uses: pat-s/[email protected]
+        with:
+          path: ~/.cache/bazel
+          key: ${{ env.cache-version }}-${{ runner.os }}-bazelisk-build-${{ hashFiles('./**') }}
+          restore-keys: ${{ env.cache-version }}-${{ runner.os }}-bazelisk-build-
+      - name: Setup
+        run: |
+          export TEST_TMPDIR=~/.cache/bazel
+          CC=g++ bazelisk test --test_output=errors //zetasql/public:sql_formatter_test
+          CC=g++ bazelisk build //zetasql/tools/zetasql-formatter:format
+          sudo cp bazel-bin/zetasql/tools/zetasql-formatter/format zetasql-formatter
+          zip zetasql-formatter_darwin_amd64.zip zetasql-formatter
+      - name: Test
+        run: |
+          cd zetasql/tools/zetasql-formatter
+          ls example_tests | xargs -n1 -I {} sh -c 'cat example_tests/{} | ../../../zetasql-formatter > example_tests_formatted/{}'
+          git diff --exit-code -- '*.sql'
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: zetasql-formatter_darwin_amd64.zip
+          prerelease: true
+          generate_release_notes: true

.gitignore

@@ -0,0 +1,4 @@
+/external
+/bazel-*
+/compile_commands.json
+/.cache/

README.md

@@ -1,89 +1,70 @@
-## ZetaSQL - Analyzer Framework for SQL
-
-ZetaSQL defines a language (grammar, types, data model, and semantics) as well
-as a parser and analyzer.  It is not itself a database or query engine. Instead
-it is intended to be used by multiple engines wanting to provide consistent
-behavior for all semantic analysis, name resolution, type checking, implicit
-casting, etc. Specific query engines may not implement all features in the
-ZetaSQL language and may give errors if specific features are not supported. For
-example, engine A may not support any updates and engine B may not support
-analytic functions.
-
-[ZetaSQL Language Guide](docs/README.md)
-
-[ZetaSQL ResolvedAST API](docs/resolved_ast.md)
-
-## Status of Project and Roadmap
-
-This codebase is being open sourced in multiple phases:
-
-1. Parser and Analyzer **Complete**
-   - Initial release includes only a subset of tests
-2. Reference Implementation **In Progress**
-   - Base capability **Complete**
-   - Function library **In Progress**
-3. Compliance Tests **Complete**
-   - includes framework for validating compliance of arbitrary engines
-4. Misc tooling
-   - Improved Formatter **In Progress**
-
-Multiplatform support is planned for the following platforms:
-
- - Linux (Ubuntu 1804 _with gcc8_ is our reference platform, but others may work).
- - MacOS (Experimental)
- - Windows (version TDB)
-
-Until all this code is released, we cannot provide any guarantees of API
-stability and cannot accept contributions. We will also be releasing more
-documentation over time, particular related to developing engines with this
-framework. Documentation on the [language](docs/) itself is fairly
-complete.
-
-
-## Flags
-ZetaSQL uses the Abseil [Flags](https://abseil.io/blog/20190509-flags) library
-to handle commandline flags. Unless otherwise documented, all flags are for
-debugging purposes only and may change, stop working or be removed at any time.
-
-
-## How to Build
-
-ZetaSQL uses [bazel](https://bazel.build) for building and dependency
-resolution. After installing bazel (we maintain support for 1.0,
-but other versions may work), simply run:
-
-```bazel build ...```
-
-## How to add as a Dependency in bazel
-See the (WORKSPACE) file, as it is a little unusual.
-
-### With docker
- TODO: Add docker build script.
-
-## Example Usage
-A very basic command line tool is available to run simple queries with the
-reference implementation:
-```bazel run //zetasql/tools/execute_query:execute_query -- "select 1 + 1;"```
-
-The reference implementation is not yet completely released and currently
-supports only a subset of functions and types.
-
-## Differential Privacy
-For questions, documentation and examples of ZetaSQLs implementation of
-Differential Privacy, please check out
-(https://github.com/google/differential-privacy).
-
-## Versions
-
-ZetaSQL makes no guarantees regarding compatibility between releases.
-Breaking changes may be made at any time. Our releases are numbered based
-on the date of the commit the release is cut from. The number format is
-YYYY.MM.n, where YYYY is the year, MM is the two digit month, and n is a
-sequence number within the time period.
+## ZetaSQL Formatter
+
+[![build](https://github.com/Matts966/zetasql-formatter/workflows/Build/badge.svg?branch=main)](https://github.com/Matts966/zetasql-formatter/actions?query=branch%main+workflow%3ABuild+)
+
+<p align="center">
+  <img src="./docs/changes.png">
+</p>
+
+This repository is forked from [google/zetasql](https://github.com/google/zetasql) and provides SQL formatter with preserved comments. This formatter can be applied to mainly BigQuery and SpanSQL.
+
+## Quick Start
+
+```bash
+# To install for MacOSX
+wget https://github.com/Matts966/zetasql-formatter/releases/latest/download/zetasql-formatter_darwin_amd64.zip \
+  && sudo unzip zetasql-formatter_darwin_amd64.zip -d /usr/local/bin
+```
+
+```bash
+# To install for Linux
+wget https://github.com/Matts966/zetasql-formatter/releases/latest/download/zetasql-formatter_linux_x86_64.zip \
+  && sudo unzip zetasql-formatter_linux_x86_64.zip -d /usr/local/bin
+```
+
+```bash
+# To apply formatter for files
+$ zetasql-formatter [files and directories]
+
+# Format stdin
+$ echo "select * from test" | zetasql-formatter
+SELECT
+  *
+FROM
+  test;
+
+$ zetasql-formatter
+select * from ok;
+-- CTRL-D
+SELECT
+  *
+FROM
+  ok;
+-- CTRL-D
+```
+
+## Integration with [efm-langserver](https://github.com/mattn/efm-langserver)
+
+- Install efm-langserver
+- Locate [`config.yaml`](https://github.com/mattn/efm-langserver#example-for-configyaml) like below
+
+```yaml
+version: 2
+tools:
+  zetasql-formatter: &zetasql-formatter
+    format-command: zetasql-formatter
+    format-stdin: true
+languages:
+  sql:
+    - <<: *zetasql-formatter
+  sql-bigquery:
+    - <<: *zetasql-formatter
+```
 
 ## License
 
 [Apache License 2.0](LICENSE)
 
-## Support Disclaimer
-This is not an officially supported Google product.
+## Sponsors
+
+The development of this formatter is sponsored by the Japan Data Science Consortium.

WORKSPACE

@@ -92,3 +92,25 @@ load("@com_google_zetasql//bazel:zetasql_deps_step_4.bzl", "zetasql_deps_step_4"
 
 zetasql_deps_step_4()
 
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+git_repository(
+    name   = "com_github_gflags_gflags",
+    remote = "https://github.com/gflags/gflags.git",
+    tag = "v2.2.2"
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+# Hedron's Compile Commands Extractor for Bazel
+# https://github.com/hedronvision/bazel-compile-commands-extractor
+http_archive(
+    name = "hedron_compile_commands",
+
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
+    # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
+    url = "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
+    strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
+)
+load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
+hedron_compile_commands_setup()

tools/BUILD

@@ -24,6 +24,7 @@ package(
 
 exports_files([
     "pom-template.xml",
+    "workspace_status.sh",
 ])
 
 bzl_library(

tools/workspace_status.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "STABLE_BUILD_GIT_DESCRIBE $(git describe --tags)"

zetasql/parser/gen_extra_files.py

@@ -70,6 +70,8 @@ class ParseTreeVisitor {
        public:
         virtual ~ParseTreeVisitor() {}
         virtual void visit(const ASTNode *node, void* data) = 0;
+        virtual void visitStart(const ASTNode *node, void* data) {};
+        virtual void visitEnd(const ASTNode *node, void* data) {};
       ''')
   for cls in concrete_classes:
     yield ('  virtual void visit{0}(const {0}* node, void* data) = 0;\n\n'
@@ -153,7 +155,9 @@ def GeneerateParseTreeAcceptMethods(
   for cls in concrete_classes:
     yield textwrap.dedent('''\
         void {0}::Accept(ParseTreeVisitor* visitor, void* data) const {{
+          visitor->visitStart(this, data);
           visitor->visit{0}(this, data);
+          visitor->visitEnd(this, data);
         }}
 
         ''').format(cls)

zetasql/parser/parser.h

@@ -22,6 +22,7 @@
 #include <utility>
 #include <variant>
 #include <vector>
+#include <deque>
 
 #include "zetasql/base/arena.h"
 #include "zetasql/parser/ast_node_kind.h"
@@ -274,6 +275,8 @@ absl::Status ParseExpression(const ParseResumeLocation& resume_location,
 // Unparse a given AST back to a canonical SQL string and return it.
 // Works for any AST node.
 std::string Unparse(const ASTNode* root);
+std::string UnparseWithComments(const ASTNode* root, std::deque<std::pair<std::string,
+                    ParseLocationPoint>>& parse_tokens);
 
 // Parse the first few keywords from <input> (ignoring whitespace, comments and
 // hints) to determine what kind of statement it is (if it is valid).