diff --git a/.github/composite-actions/compile-antlr/action.yml b/.github/composite-actions/compile-antlr/action.yml index ea5fc32536..28970b5468 100644 --- a/.github/composite-actions/compile-antlr/action.yml +++ b/.github/composite-actions/compile-antlr/action.yml @@ -12,7 +12,7 @@ runs: - name: Compile ANTLR run: | - ANTLR_VERSION=4.9.3 + ANTLR_VERSION=4.13.2 cd contrib/babelfishpg_tsql/antlr/thirdparty/antlr/ sudo cp "antlr-$ANTLR_VERSION-complete.jar" /usr/local/lib diff --git a/.github/composite-actions/setup-new-version/action.yml b/.github/composite-actions/setup-new-version/action.yml new file mode 100644 index 0000000000..6408999427 --- /dev/null +++ b/.github/composite-actions/setup-new-version/action.yml @@ -0,0 +1,80 @@ +name: 'Setup Base Version' +inputs: + engine_branch: + description: "Engine Branch" + required: true + extension_branch: + description: "Extension Branch" + required: true + pg_new_dir: + description: "Install new version in this directory" + required: true + +runs: + using: "composite" + steps: + - name: Build Modified Postgres using latest version + id: build-modified-postgres-new + if: always() + uses: ./.github/composite-actions/build-modified-postgres + with: + engine_branch: ${{ inputs.engine_branch }} + install_dir: ${{ inputs.pg_new_dir }} + + - name: Copy ANTLR + id: copy-antlr + if: always() && steps.build-modified-postgres-new.outcome == 'success' + run: cp "/usr/local/lib/libantlr4-runtime.so.4.13.2" ~/${{ inputs.pg_new_dir }}/lib/ + shell: bash + + - name: Build Extensions + id: build-extensions-new + if: always() && steps.copy-antlr.outcome == 'success' + uses: ./.github/composite-actions/build-extensions + with: + install_dir: ${{ inputs.pg_new_dir }} + extension_branch: ${{ inputs.extension_branch }} + + - uses: actions/checkout@v2 + + - name: Build tds_fdw Extension + id: build-tds_fdw-extension + if: always() && steps.build-extensions-new.outcome == 'success' + uses: ./.github/composite-actions/build-tds_fdw-extension + with: + install_dir: ${{ inputs.pg_new_dir }} + + - name: Build vector Extension + id: build-vector-extension + if: always() && steps.build-tds_fdw-extension.outcome == 'success' + uses: ./.github/composite-actions/build-vector-extension + with: + install_dir: ${{ inputs.pg_new_dir }} + + - name: Build PostGIS Extension + id: build-postgis-extension + if: always() && steps.build-vector-extension.outcome == 'success' + uses: ./.github/composite-actions/build-postgis-extension + with: + install_dir: ${{ inputs.pg_new_dir }} + + - name: Setup new data directory + id: setup-new-datadir + if: always() && steps.build-postgis-extension.outcome == 'success' + run: | + cd ~ + ~/${{ inputs.pg_new_dir }}/bin/initdb -D ~/${{ inputs.pg_new_dir }}/data + cd ~/${{ inputs.pg_new_dir }}/data + sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf + sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf + ipaddress=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{ print $2}') + # Allow only runner to have trust authentication, all other users must provide a password + { + sudo echo "local all runner trust" + sudo echo "local all all md5" + sudo echo "host all runner 127.0.0.1/32 trust" + sudo echo "host all runner $ipaddress/32 trust" + sudo echo "host all all 0.0.0.0/0 md5" + sudo echo "host all all ::/0 md5" + } > pg_hba.conf + shell: bash diff --git a/.github/workflows/isolation-tests.yml b/.github/workflows/isolation-tests.yml index dafbd7fef4..68b6dff152 100644 --- a/.github/workflows/isolation-tests.yml +++ b/.github/workflows/isolation-tests.yml @@ -49,12 +49,12 @@ jobs: curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list cd ~/work/babelfish_extensions/babelfish_extensions/test/python sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 python3-dev - pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.9.3 + pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.13.2 - name: Generate .spec file parser run: | cd ~/work/babelfish_extensions/babelfish_extensions/test/python/isolationtest/ - java -Xmx500M -cp /usr/local/lib/antlr-4.9.3-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener + java -Xmx500M -cp /usr/local/lib/antlr-4.13.2-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener - name: Run Isolation tests run: | diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml index e2a2764c67..bcbb69d6a3 100644 --- a/.github/workflows/major-version-upgrade.yml +++ b/.github/workflows/major-version-upgrade.yml @@ -90,7 +90,7 @@ jobs: install_dir: ${{env.NEW_INSTALL_DIR}} - name: Copy ANTLR - run: cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" ~/${{env.NEW_INSTALL_DIR}}/lib/ + run: cp "/usr/local/lib/libantlr4-runtime.so.4.13.2" ~/${{env.NEW_INSTALL_DIR}}/lib/ - name: Build Extensions using latest version id: build-extensions-new diff --git a/INSTALLING.md.tmpl b/INSTALLING.md.tmpl index 1b7c578f90..2cbe91e11b 100644 --- a/INSTALLING.md.tmpl +++ b/INSTALLING.md.tmpl @@ -53,19 +53,19 @@ export PG_SRC=$(realpath $PWD) ### Compile ANTLR 4 -Unfortunately, there are [no prebuilt C++ binaries for the Antlr 4.9.3 runtime version](https://www.antlr.org/download.html) for Linux. You will need to compile and install ANTLR manually. +Unfortunately, there are [no prebuilt C++ binaries for the Antlr 4.13.2 runtime version](https://www.antlr.org/download.html) for Linux. You will need to compile and install ANTLR manually. First, define the following variables in your environment: ```sh -export ANTLR4_VERSION=4.9.3 +export ANTLR4_VERSION=4.13.2 export ANTLR4_JAVA_BIN=/usr/bin/java export ANTLR4_RUNTIME_LIBRARIES=/usr/include/antlr4-runtime export ANTLR_EXECUTABLE=/usr/local/lib/antlr-${ANTLR4_VERSION}-complete.jar export ANTLR_RUNTIME=~/antlr4 ``` -The [Antlr 4.9.3 Runtime](https://www.antlr.org/) files are distributed with the Babelfish source code. Use the following commands to copy the files into place: +The [Antlr 4.13.2 Runtime](https://www.antlr.org/) files are distributed with the Babelfish source code. Use the following commands to copy the files into place: ```sh sudo cp ${PG_SRC}/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-${ANTLR4_VERSION}-complete.jar /usr/local/lib diff --git a/contrib/README.md b/contrib/README.md index 2a37460335..2ebb0f45cc 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -72,30 +72,30 @@ The following build instructions comply with Ubuntu 20.04 and Amazon Linux 2 env 2. Install ANTLR - The `babelfishpg_tsql` contrib has a plug-in parser generated by ANTLR, which depends on cmake and `antlr4-cpp-runtime-4.9.3`. Unfortunately, there aren't binaries for [C++ targets available](https://www.antlr.org/download.html). You'll also need have `uuid-devel` installed in order to install `antlr4-cpp-runtime-4.9.3`. + The `babelfishpg_tsql` contrib has a plug-in parser generated by ANTLR, which depends on cmake and `antlr4-cpp-runtime-4.13.2`. Unfortunately, there aren't binaries for [C++ targets available](https://www.antlr.org/download.html). You'll also need have `uuid-devel` installed in order to install `antlr4-cpp-runtime-4.13.2`. First copy the jar file in `contrib/babelfishpg_tsql/antlr/thirdparty/antlr/` to another location: ``` cd babelfish_extensions/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/ - sudo cp antlr-4.9.3-complete.jar /usr/local/lib + sudo cp antlr-4.13.2-complete.jar /usr/local/lib ``` Compile antlr4: ``` - wget http://www.antlr.org/download/antlr4-cpp-runtime-4.9.3-source.zip - unzip -d antlr4 antlr4-cpp-runtime-4.9.3-source.zip + wget http://www.antlr.org/download/antlr4-cpp-runtime-4.13.2-source.zip + unzip -d antlr4 antlr4-cpp-runtime-4.13.2-source.zip cd antlr4 mkdir build && cd build - cmake .. -DANTLR_JAR_LOCATION=/usr/local/lib/antlr-4.9.3-complete.jar -DCMAKE_INSTALL_PREFIX=/usr/local -DWITH_DEMO=True + cmake .. -DANTLR_JAR_LOCATION=/usr/local/lib/antlr-4.13.2-complete.jar -DCMAKE_INSTALL_PREFIX=/usr/local -DWITH_DEMO=True make sudo make install ``` Copy libantlr4-runtime to postgres/lib ``` - cp /usr/local/lib/libantlr4-runtime.so.4.9.3 ~/postgres/lib/ + cp /usr/local/lib/libantlr4-runtime.so.4.13.2 ~/postgres/lib/ ``` - If you come across the error diff --git a/contrib/babelfishpg_tsql/Makefile b/contrib/babelfishpg_tsql/Makefile index 8c3bbee0ea..9ae4f21273 100644 --- a/contrib/babelfishpg_tsql/Makefile +++ b/contrib/babelfishpg_tsql/Makefile @@ -84,6 +84,7 @@ PG_CXXFLAGS += -Wno-deprecated -Wno-error=attributes -Wno-suggest-attribute=form PG_CXXFLAGS += -Wno-undef -Wall -Wcpp PG_CXXFLAGS += -Wno-register # otherwise C++17 gags on PostgreSQL headers PG_CXXFLAGS += -I$(ANTLR4_RUNTIME_INCLUDE_DIR) +PG_CXXFLAGS += -std=c++17 PG_CFLAGS += -g -Werror -Wfloat-conversion PG_CFLAGS += -fstack-protector-strong PG_CPPFLAGS += -I$(TSQLSRC) -I$(PG_SRC) -DFAULT_INJECTOR -Wfloat-conversion diff --git a/contrib/babelfishpg_tsql/antlr/CMakeLists.txt b/contrib/babelfishpg_tsql/antlr/CMakeLists.txt index 4c46475532..926dc7d518 100644 --- a/contrib/babelfishpg_tsql/antlr/CMakeLists.txt +++ b/contrib/babelfishpg_tsql/antlr/CMakeLists.txt @@ -3,8 +3,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.7 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake-dir) -# compiler must be 11 or 14 -set(CMAKE_CXX_STANDARD 14) +# compiler must be 17+ to support ANTLR-4.13 +set(CMAKE_CXX_STANDARD 17) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -O2 -ggdb -w -Wno-deprecated") set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -fPIC -O2 -ggdb -w -Wno-deprecated") @@ -17,7 +17,7 @@ include_directories(${MYDIR}) # set variable pointing to the antlr tool that supports C++ # this is not required if the jar file can be found under PATH environment -set(ANTLR_EXECUTABLE ${PROJECT_SOURCE_DIR}/thirdparty/antlr/antlr-4.9.3-complete.jar) +set(ANTLR_EXECUTABLE ${PROJECT_SOURCE_DIR}/thirdparty/antlr/antlr-4.13.2-complete.jar) # add macros to generate ANTLR Cpp code from grammar find_package(ANTLR REQUIRED) diff --git a/contrib/babelfishpg_tsql/antlr/cmake-dir/FindANTLR.cmake b/contrib/babelfishpg_tsql/antlr/cmake-dir/FindANTLR.cmake index 968feded7a..185ea3cd2d 100644 --- a/contrib/babelfishpg_tsql/antlr/cmake-dir/FindANTLR.cmake +++ b/contrib/babelfishpg_tsql/antlr/cmake-dir/FindANTLR.cmake @@ -2,7 +2,7 @@ if(NOT ANTLR_EXECUTABLE) find_program(ANTLR_EXECUTABLE - NAMES antlr.jar antlr4.jar antlr-4.jar antlr-4.9.3-complete.jar) + NAMES antlr.jar antlr4.jar antlr-4.jar antlr-4.13.2-complete.jar) endif() set(Java_JAVA_EXECUTABLE $ENV{ANTLR4_JAVA_BIN}) @@ -18,7 +18,7 @@ if(ANTLR_EXECUTABLE AND Java_JAVA_EXECUTABLE) OUTPUT_STRIP_TRAILING_WHITESPACE) if(ANTLR_COMMAND_RESULT EQUAL 0) - string(REGEX MATCH "Version [0-9]+(\\.[0-9])*" ANTLR_VERSION ${ANTLR_COMMAND_OUTPUT}) + string(REGEX MATCH "Version [0-9]+(\\.[0-9]+)*" ANTLR_VERSION ${ANTLR_COMMAND_OUTPUT}) string(REPLACE "Version " "" ANTLR_VERSION ${ANTLR_VERSION}) else() message( diff --git a/contrib/babelfishpg_tsql/antlr/cmake-dir/README.md b/contrib/babelfishpg_tsql/antlr/cmake-dir/README.md index 0ebe1dd51e..3a4436258a 100644 --- a/contrib/babelfishpg_tsql/antlr/cmake-dir/README.md +++ b/contrib/babelfishpg_tsql/antlr/cmake-dir/README.md @@ -31,7 +31,7 @@ include_directories(${ANTLR4_INCLUDE_DIRS}) # set variable pointing to the antlr tool that supports C++ # this is not required if the jar file can be found under PATH environment -set(ANTLR_EXECUTABLE /home/user/antlr-4.9.3-complete.jar) +set(ANTLR_EXECUTABLE /home/user/antlr-4.13.2-complete.jar) # add macros to generate ANTLR Cpp code from grammar find_package(ANTLR REQUIRED) diff --git a/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-4.13.2-complete.jar b/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-4.13.2-complete.jar new file mode 100644 index 0000000000..75bfcc3976 Binary files /dev/null and b/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-4.13.2-complete.jar differ diff --git a/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-4.9.3-complete.jar b/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-4.9.3-complete.jar deleted file mode 100644 index 749296fe7b..0000000000 Binary files a/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-4.9.3-complete.jar and /dev/null differ diff --git a/contrib/babelfishpg_tsql/src/tsqlIface.cpp b/contrib/babelfishpg_tsql/src/tsqlIface.cpp index 99b3caeb63..b23c550295 100644 --- a/contrib/babelfishpg_tsql/src/tsqlIface.cpp +++ b/contrib/babelfishpg_tsql/src/tsqlIface.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #pragma GCC diagnostic ignored "-Wattributes" @@ -10,6 +11,7 @@ #include "antlr4-runtime.h" // antlr4-cpp-runtime #include "tree/ParseTreeWalker.h" // antlr4-cpp-runtime #include "tree/ParseTreeProperty.h" // antlr4-cpp-runtime +#include "support/Utf8.h" #include "../antlr/antlr4cpp_generated_src/TSqlLexer/TSqlLexer.h" #include "../antlr/antlr4cpp_generated_src/TSqlParser/TSqlParser.h" @@ -471,7 +473,7 @@ format_errmsg(const char *fmt, const char *arg1, const char *arg2); inline std::u32string utf8_to_utf32(const char* s) { - return antlrcpp::utf8_to_utf32(s, s + strlen(s)); + return antlrcpp::Utf8::lenientDecode(std::string_view(s, strlen(s))); } class MyInputStream : public ANTLRInputStream @@ -485,7 +487,7 @@ class MyInputStream : public ANTLRInputStream void setText(size_t pos, const char *newText) { - UTF32String newText32 = utf8_to_utf32(newText); + std::u32string newText32 = utf8_to_utf32(newText); _data.replace(pos, newText32.size(), newText32); } @@ -566,9 +568,9 @@ void PLtsql_expr_query_mutator::run() } if (cursor < strlen(expr->query)) rewritten_query += query.substr(cursor); // copy remaining expr->query - - // update query string with quoted one - std::string new_query = antlrcpp::utf32_to_utf8(rewritten_query); + + // update query string + std::string new_query = antlrcpp::Utf8::lenientEncode(rewritten_query); expr->query = pstrdup(new_query.c_str()); } diff --git a/dev-tools.sh b/dev-tools.sh index 6166e14acf..5afec28192 100755 --- a/dev-tools.sh +++ b/dev-tools.sh @@ -167,7 +167,7 @@ init_pg() { make -j 4 make install cd contrib && make && sudo make install - cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" $2/postgres/lib/ + cp "/usr/local/lib/libantlr4-runtime.so.4.13.2" $2/postgres/lib/ init_pghint $1 $2 } diff --git a/test/python/isolationtest/specParserVisitorImpl.py b/test/python/isolationtest/specParserVisitorImpl.py index b376331cb2..e40a063149 100644 --- a/test/python/isolationtest/specParserVisitorImpl.py +++ b/test/python/isolationtest/specParserVisitorImpl.py @@ -2,7 +2,7 @@ from .parser.specParserVisitor import specParserVisitor -# Generated from specParser.g4 by ANTLR 4.9.3 +# Generated from specParser.g4 by ANTLR 4.13.2 from antlr4 import * from .parser.specParser import specParser