Update antlr to 4.13.2

Signed-off-by: Jason Teng <[email protected]>

.github/composite-actions/compile-antlr/action.yml

@@ -12,7 +12,7 @@ runs:
 
     - name: Compile ANTLR
       run: |
-        ANTLR_VERSION=4.9.3
+        ANTLR_VERSION=4.13.2
         
         cd contrib/babelfishpg_tsql/antlr/thirdparty/antlr/
         sudo cp "antlr-$ANTLR_VERSION-complete.jar" /usr/local/lib

.github/composite-actions/setup-new-version/action.yml

@@ -0,0 +1,80 @@
+name: 'Setup Base Version'
+inputs:
+  engine_branch: 
+    description: "Engine Branch"
+    required: true
+  extension_branch: 
+    description: "Extension Branch"
+    required: true
+  pg_new_dir:
+    description: "Install new version in this directory"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Build Modified Postgres using latest version
+      id: build-modified-postgres-new
+      if: always()
+      uses: ./.github/composite-actions/build-modified-postgres
+      with:
+        engine_branch: ${{ inputs.engine_branch }}
+        install_dir: ${{ inputs.pg_new_dir }}
+
+    - name: Copy ANTLR
+      id: copy-antlr
+      if: always() && steps.build-modified-postgres-new.outcome == 'success'
+      run: cp "/usr/local/lib/libantlr4-runtime.so.4.13.2" ~/${{ inputs.pg_new_dir }}/lib/
+      shell: bash
+
+    - name: Build Extensions
+      id: build-extensions-new
+      if: always() && steps.copy-antlr.outcome == 'success'
+      uses: ./.github/composite-actions/build-extensions
+      with:
+        install_dir: ${{ inputs.pg_new_dir }}
+        extension_branch: ${{ inputs.extension_branch }}
+
+    - uses: actions/checkout@v2
+
+    - name: Build tds_fdw Extension
+      id: build-tds_fdw-extension
+      if: always() && steps.build-extensions-new.outcome == 'success'
+      uses: ./.github/composite-actions/build-tds_fdw-extension
+      with:
+        install_dir: ${{ inputs.pg_new_dir }}
+
+    - name: Build vector Extension
+      id: build-vector-extension
+      if: always() && steps.build-tds_fdw-extension.outcome == 'success'
+      uses: ./.github/composite-actions/build-vector-extension
+      with:
+        install_dir: ${{ inputs.pg_new_dir }}
+
+    - name: Build PostGIS Extension
+      id: build-postgis-extension
+      if: always() && steps.build-vector-extension.outcome == 'success'
+      uses: ./.github/composite-actions/build-postgis-extension
+      with:
+        install_dir: ${{ inputs.pg_new_dir }}
+
+    - name: Setup new data directory
+      id: setup-new-datadir
+      if: always() && steps.build-postgis-extension.outcome == 'success'
+      run: |
+        cd ~
+        ~/${{ inputs.pg_new_dir }}/bin/initdb -D ~/${{ inputs.pg_new_dir }}/data
+        cd ~/${{ inputs.pg_new_dir }}/data
+        sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf
+        sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf
+        ipaddress=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{ print $2}')
+        # Allow only runner to have trust authentication, all other users must provide a password
+        {
+          sudo echo "local   all             runner                                   trust"
+          sudo echo "local   all             all                                      md5"
+          sudo echo "host    all             runner          127.0.0.1/32             trust"
+          sudo echo "host    all             runner          $ipaddress/32            trust"
+          sudo echo "host    all             all             0.0.0.0/0                md5"
+          sudo echo "host    all             all             ::/0                     md5"
+        } > pg_hba.conf
+      shell: bash

.github/workflows/isolation-tests.yml

@@ -49,12 +49,12 @@ jobs:
           curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
           cd ~/work/babelfish_extensions/babelfish_extensions/test/python
           sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 python3-dev
-          pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.9.3
+          pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.13.2
       
       - name: Generate .spec file parser
         run: |
           cd ~/work/babelfish_extensions/babelfish_extensions/test/python/isolationtest/
-          java -Xmx500M -cp /usr/local/lib/antlr-4.9.3-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener
+          java -Xmx500M -cp /usr/local/lib/antlr-4.13.2-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener
 
       - name: Run Isolation tests
         run: | 

.github/workflows/major-version-upgrade.yml

@@ -90,7 +90,7 @@ jobs:
           install_dir: ${{env.NEW_INSTALL_DIR}}
 
       - name: Copy ANTLR
-        run: cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" ~/${{env.NEW_INSTALL_DIR}}/lib/
+        run: cp "/usr/local/lib/libantlr4-runtime.so.4.13.2" ~/${{env.NEW_INSTALL_DIR}}/lib/
 
       - name: Build Extensions using latest version
         id: build-extensions-new

INSTALLING.md.tmpl

@@ -53,19 +53,19 @@ export PG_SRC=$(realpath $PWD)
 
 ### Compile ANTLR 4
 
-Unfortunately, there are [no prebuilt C++ binaries for the Antlr 4.9.3 runtime version](https://www.antlr.org/download.html) for Linux. You will need to compile and install ANTLR manually.
+Unfortunately, there are [no prebuilt C++ binaries for the Antlr 4.13.2 runtime version](https://www.antlr.org/download.html) for Linux. You will need to compile and install ANTLR manually.
 
 First, define the following variables in your environment:
 
 ```sh
-export ANTLR4_VERSION=4.9.3
+export ANTLR4_VERSION=4.13.2
 export ANTLR4_JAVA_BIN=/usr/bin/java
 export ANTLR4_RUNTIME_LIBRARIES=/usr/include/antlr4-runtime
 export ANTLR_EXECUTABLE=/usr/local/lib/antlr-${ANTLR4_VERSION}-complete.jar
 export ANTLR_RUNTIME=~/antlr4
 ```
 
-The [Antlr 4.9.3 Runtime](https://www.antlr.org/) files are distributed with the Babelfish source code.  Use the following commands to copy the files into place:
+The [Antlr 4.13.2 Runtime](https://www.antlr.org/) files are distributed with the Babelfish source code.  Use the following commands to copy the files into place:
 
 ```sh
 sudo cp ${PG_SRC}/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-${ANTLR4_VERSION}-complete.jar /usr/local/lib

contrib/README.md

@@ -72,30 +72,30 @@ The following build instructions comply with Ubuntu 20.04 and Amazon Linux 2 env
 
 2. Install ANTLR
 
-    The `babelfishpg_tsql` contrib has a plug-in parser generated by ANTLR, which depends on cmake and `antlr4-cpp-runtime-4.9.3`. Unfortunately, there aren't binaries for [C++ targets available](https://www.antlr.org/download.html). You'll also need have `uuid-devel` installed in order to install `antlr4-cpp-runtime-4.9.3`.
+    The `babelfishpg_tsql` contrib has a plug-in parser generated by ANTLR, which depends on cmake and `antlr4-cpp-runtime-4.13.2`. Unfortunately, there aren't binaries for [C++ targets available](https://www.antlr.org/download.html). You'll also need have `uuid-devel` installed in order to install `antlr4-cpp-runtime-4.13.2`.
 
     First copy the jar file in `contrib/babelfishpg_tsql/antlr/thirdparty/antlr/` to another location:
 
     ```
     cd babelfish_extensions/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/
-    sudo cp antlr-4.9.3-complete.jar /usr/local/lib
+    sudo cp antlr-4.13.2-complete.jar /usr/local/lib
     ```
 
     Compile antlr4:
 
     ```
-    wget http://www.antlr.org/download/antlr4-cpp-runtime-4.9.3-source.zip
-    unzip -d antlr4 antlr4-cpp-runtime-4.9.3-source.zip 
+    wget http://www.antlr.org/download/antlr4-cpp-runtime-4.13.2-source.zip
+    unzip -d antlr4 antlr4-cpp-runtime-4.13.2-source.zip 
     cd antlr4
     mkdir build && cd build 
-    cmake .. -DANTLR_JAR_LOCATION=/usr/local/lib/antlr-4.9.3-complete.jar -DCMAKE_INSTALL_PREFIX=/usr/local -DWITH_DEMO=True
+    cmake .. -DANTLR_JAR_LOCATION=/usr/local/lib/antlr-4.13.2-complete.jar -DCMAKE_INSTALL_PREFIX=/usr/local -DWITH_DEMO=True
     make
     sudo make install
     ```
 
     Copy libantlr4-runtime to postgres/lib
     ```
-    cp /usr/local/lib/libantlr4-runtime.so.4.9.3 ~/postgres/lib/
+    cp /usr/local/lib/libantlr4-runtime.so.4.13.2 ~/postgres/lib/
     ```
 
     - If you come across the error

contrib/babelfishpg_tsql/Makefile

@@ -84,6 +84,7 @@ PG_CXXFLAGS += -Wno-deprecated -Wno-error=attributes -Wno-suggest-attribute=form
 PG_CXXFLAGS += -Wno-undef -Wall -Wcpp
 PG_CXXFLAGS += -Wno-register # otherwise C++17 gags on PostgreSQL headers
 PG_CXXFLAGS += -I$(ANTLR4_RUNTIME_INCLUDE_DIR)
+PG_CXXFLAGS += -std=c++17
 PG_CFLAGS += -g -Werror -Wfloat-conversion
 PG_CFLAGS += -fstack-protector-strong
 PG_CPPFLAGS += -I$(TSQLSRC) -I$(PG_SRC) -DFAULT_INJECTOR -Wfloat-conversion

contrib/babelfishpg_tsql/antlr/CMakeLists.txt

@@ -3,8 +3,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.7 FATAL_ERROR)
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake-dir)
 
-# compiler must be 11 or 14
-set(CMAKE_CXX_STANDARD 14)
+# compiler must be 17+ to support ANTLR-4.13
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -O2 -ggdb -w -Wno-deprecated")
 set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -fPIC -O2 -ggdb -w -Wno-deprecated")
 
@@ -17,7 +17,7 @@ include_directories(${MYDIR})
 
 # set variable pointing to the antlr tool that supports C++
 # this is not required if the jar file can be found under PATH environment
-set(ANTLR_EXECUTABLE ${PROJECT_SOURCE_DIR}/thirdparty/antlr/antlr-4.9.3-complete.jar)
+set(ANTLR_EXECUTABLE ${PROJECT_SOURCE_DIR}/thirdparty/antlr/antlr-4.13.2-complete.jar)
 # add macros to generate ANTLR Cpp code from grammar
 find_package(ANTLR REQUIRED)
 

contrib/babelfishpg_tsql/antlr/cmake-dir/FindANTLR.cmake

@@ -2,7 +2,7 @@
 
 if(NOT ANTLR_EXECUTABLE)
   find_program(ANTLR_EXECUTABLE
-               NAMES antlr.jar antlr4.jar antlr-4.jar antlr-4.9.3-complete.jar)
+               NAMES antlr.jar antlr4.jar antlr-4.jar antlr-4.13.2-complete.jar)
 endif()
 
 set(Java_JAVA_EXECUTABLE $ENV{ANTLR4_JAVA_BIN})
@@ -18,7 +18,7 @@ if(ANTLR_EXECUTABLE AND Java_JAVA_EXECUTABLE)
       OUTPUT_STRIP_TRAILING_WHITESPACE)
 
   if(ANTLR_COMMAND_RESULT EQUAL 0)
-    string(REGEX MATCH "Version [0-9]+(\\.[0-9])*" ANTLR_VERSION ${ANTLR_COMMAND_OUTPUT})
+    string(REGEX MATCH "Version [0-9]+(\\.[0-9]+)*" ANTLR_VERSION ${ANTLR_COMMAND_OUTPUT})
     string(REPLACE "Version " "" ANTLR_VERSION ${ANTLR_VERSION})
   else()
     message(

contrib/babelfishpg_tsql/antlr/cmake-dir/README.md

@@ -31,7 +31,7 @@ include_directories(${ANTLR4_INCLUDE_DIRS})
 
 # set variable pointing to the antlr tool that supports C++
 # this is not required if the jar file can be found under PATH environment
-set(ANTLR_EXECUTABLE /home/user/antlr-4.9.3-complete.jar)
+set(ANTLR_EXECUTABLE /home/user/antlr-4.13.2-complete.jar)
 # add macros to generate ANTLR Cpp code from grammar
 find_package(ANTLR REQUIRED)
 

contrib/babelfishpg_tsql/src/tsqlIface.cpp

@@ -3,13 +3,15 @@
 #include <iostream>
 #include <strstream>
 #include <string>
+#include <string_view>
 #include <unordered_map>
 
 #pragma GCC diagnostic ignored "-Wattributes"
 
 #include "antlr4-runtime.h" // antlr4-cpp-runtime
 #include "tree/ParseTreeWalker.h" // antlr4-cpp-runtime
 #include "tree/ParseTreeProperty.h" // antlr4-cpp-runtime
+#include "support/Utf8.h"
 
 #include "../antlr/antlr4cpp_generated_src/TSqlLexer/TSqlLexer.h"
 #include "../antlr/antlr4cpp_generated_src/TSqlParser/TSqlParser.h"
@@ -471,7 +473,7 @@ format_errmsg(const char *fmt, const char *arg1, const char *arg2);
 
 inline std::u32string utf8_to_utf32(const char* s)
 {
-	return antlrcpp::utf8_to_utf32(s, s + strlen(s));
+	return antlrcpp::Utf8::lenientDecode(std::string_view(s, strlen(s)));
 }
 
 class MyInputStream : public ANTLRInputStream
@@ -485,7 +487,7 @@ class MyInputStream : public ANTLRInputStream
 
 		void setText(size_t pos, const char *newText)
 		{
-			UTF32String	newText32 = utf8_to_utf32(newText);
+			std::u32string	newText32 = utf8_to_utf32(newText);
 
 			_data.replace(pos, newText32.size(), newText32);
 		}
@@ -566,9 +568,9 @@ void PLtsql_expr_query_mutator::run()
 	}
 	if (cursor < strlen(expr->query))
 		rewritten_query += query.substr(cursor); // copy remaining expr->query
-
-	// update query string with quoted one
-	std::string new_query = antlrcpp::utf32_to_utf8(rewritten_query);
+		
+	// update query string
+	std::string new_query = antlrcpp::Utf8::lenientEncode(rewritten_query);
 	expr->query = pstrdup(new_query.c_str());
 }
 

dev-tools.sh

@@ -167,7 +167,7 @@ init_pg() {
     make -j 4
     make install
     cd contrib && make && sudo make install
-    cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" $2/postgres/lib/
+    cp "/usr/local/lib/libantlr4-runtime.so.4.13.2" $2/postgres/lib/
     init_pghint $1 $2
 }
 

test/python/isolationtest/specParserVisitorImpl.py

@@ -2,7 +2,7 @@
 from .parser.specParserVisitor import specParserVisitor
 
 
-# Generated from specParser.g4 by ANTLR 4.9.3
+# Generated from specParser.g4 by ANTLR 4.13.2
 from antlr4 import *
 from .parser.specParser import specParser