PR Code Coverage with Coveralls (babelfish-for-postgresql#2205)

urrently we run code coverage as scheduled job on a daily basis where all the tests are run one after the another. We aim to generate coverage for pull request as well. But running all the test frameworks sequentially takes too long. So we create a new workflow which will trigger all the tests in parallel and merge the individual code coverage reports into one overall code coverage report. Specifically we are merging the .info files generated by lcov. These files contains the count of times each line / function has been hit & the relative path for this line / function in source code. So as long as we use the same source code, merging these files will be okay. The consolidated lcov.info is uploaded to Coveralls - A web-based code coverage service that provided useful insights about the how well the newly introduced code diff in a pull request is covered by the test suites. Task: BABEL-4602 Co-authored-by: Tanzeel Khan [email protected], Sharu Goel [email protected] Signed-off-by: Sharu Goel [email protected]
amazon-aurora · Jan 8, 2024 · a75d3ef · a75d3ef
1 parent d4bedd4
commit a75d3ef
Show file tree

Hide file tree

Showing 9 changed files with 278 additions and 18 deletions.
diff --git a/.github/composite-actions/build-modified-postgres/action.yml b/.github/composite-actions/build-modified-postgres/action.yml
@@ -51,12 +51,10 @@ runs:
           ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu --enable-tap-tests --with-gssapi
         elif [[ ${{inputs.code_coverage}} == "yes" ]]; then
           ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-coverage --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu
-        else
-          if [[ ${{inputs.release_mode}} == "yes" ]]; then
+        elif [[ ${{inputs.release_mode}} == "yes" ]]; then
             ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 CFLAGS="-ggdb -O2" --with-libxml --with-uuid=ossp --with-icu
-          else
+        else
             ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu
-          fi
         fi
         make -j 4 2>error.txt
         make install

diff --git a/.github/composite-actions/install-dependencies/action.yml b/.github/composite-actions/install-dependencies/action.yml
@@ -11,6 +11,7 @@ runs:
         sudo apt-get update --fix-missing -y
         sudo apt-get install uuid-dev openjdk-8-jre libicu-dev libxml2-dev openssl libssl-dev python3-dev libossp-uuid-dev libpq-dev cmake pkg-config g++ build-essential bison mssql-tools unixodbc-dev libsybdb5 freetds-dev freetds-common gdal-bin libgdal-dev libgeos-dev gdb
         sudo apt install -y ccache
+        sudo apt-get install lcov
         sudo /usr/sbin/update-ccache-symlinks
         echo 'export PATH="/usr/lib/ccache:$PATH"' | tee -a ~/.bashrc
         source ~/.bashrc && echo $PATH

diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
@@ -17,12 +17,6 @@ jobs:
         if: always()
         uses: ./.github/composite-actions/install-dependencies
 
-      - name: Install Code Coverage Dependencies
-        id: install-code-coverage-dependencies
-        if: always()
-        run: |
-          sudo apt-get install lcov
-
       - name: Build Modified Postgres
         id: build-modified-postgres
         if: always() && steps.install-dependencies.outcome == 'success'

diff --git a/.github/workflows/dotnet-tests.yml b/.github/workflows/dotnet-tests.yml
@@ -1,5 +1,5 @@
 name: Dotnet Framework Tests
-on: [push, pull_request]
+on: [workflow_call]
 
 jobs:
   run-babelfish-dotnet-tests:
@@ -17,6 +17,9 @@ jobs:
         id: build-modified-postgres
         if: always() && steps.install-dependencies.outcome == 'success'
         uses: ./.github/composite-actions/build-modified-postgres
+        with:
+          install_dir: 'psql'
+          code_coverage: 'yes'
 
       - name: Compile ANTLR
         id: compile-antlr
@@ -42,3 +45,35 @@ jobs:
         id: run-dotnet-tests
         if: always() && steps.install-extensions.outcome == 'success'
         uses: ./.github/composite-actions/install-and-run-dotnet
+
+      - name: Generate Code Coverage
+        id: generate-code-coverage
+        if: always() && steps.run-dotnet-tests.outcome == 'success'
+        run: |
+          export PG_CONFIG=~/psql/bin/pg_config
+          export PG_SRC=~/work/postgresql_modified_for_babelfish
+          export cmake=$(which cmake)
+          cd contrib
+          for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql  
+          do
+            cd $ext
+            /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info
+            cd ..
+          done
+        shell: bash
+
+      - name: Summarize code coverage
+        id: code-coverage-summary
+        if: always() && steps.generate-code-coverage.outcome == 'success'
+        run: |
+          cd contrib/
+          lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o dotnet-lcov.info
+          lcov --list dotnet-lcov.info
+
+      - name: Upload Coverage Report for Babelfish Extensions
+        if: always() && steps.code-coverage-summary.outcome == 'success'
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-babelfish-extensions-dotnet
+          path: contrib/dotnet-lcov.info
+          retention-days: 1
diff --git a/.github/workflows/isolation-tests.yml b/.github/workflows/isolation-tests.yml
@@ -1,5 +1,5 @@
 name: Babelfish Smoke Tests
-on: [push, pull_request]
+on: [workflow_call]
 
 jobs:
   isolation-tests:
@@ -18,6 +18,9 @@ jobs:
         id: build-modified-postgres
         if: always() && steps.install-dependencies.outcome == 'success'
         uses: ./.github/composite-actions/build-modified-postgres
+        with:
+          install_dir: 'psql'
+          code_coverage: 'yes'
 
       - name: Compile ANTLR
         id: compile-antlr
@@ -58,6 +61,7 @@ jobs:
           java -Xmx500M -cp /usr/local/lib/antlr-4.9.3-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener
 
       - name: Run Isolation tests
+        id: run-isolation-tests
         run: | 
           cd test/python
           compareWithFile=true \
@@ -74,3 +78,35 @@ jobs:
             runIsolationTests=true \
             stepTimeLimit=30 \
             pytest -s --tb=long -q .
+        
+      - name: Generate Code Coverage
+        id: generate-code-coverage
+        if: always() && steps.run-isolation-tests.outcome == 'success'
+        run: |
+          export PG_CONFIG=~/psql/bin/pg_config
+          export PG_SRC=~/work/postgresql_modified_for_babelfish
+          export cmake=$(which cmake)
+          cd contrib
+          for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql  
+          do
+            cd $ext
+            /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info
+            cd ..
+          done
+        shell: bash
+
+      - name: Summarize code coverage
+        id: code-coverage-summary
+        if: always() && steps.generate-code-coverage.outcome == 'success'
+        run: |
+          cd contrib/
+          lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o isolation-lcov.info
+          lcov --list isolation-lcov.info
+
+      - name: Upload Coverage Report for Babelfish Extensions
+        if: always() && steps.code-coverage-summary.outcome == 'success'
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-babelfish-extensions-isolation
+          path: contrib/isolation-lcov.info
+          retention-days: 1
diff --git a/.github/workflows/jdbc-tests.yml b/.github/workflows/jdbc-tests.yml
@@ -1,5 +1,5 @@
 name: JDBC Tests
-on: [push, pull_request]
+on: [workflow_call]
 
 jobs:
   run-babelfish-jdbc-tests:
@@ -19,6 +19,9 @@ jobs:
         id: build-modified-postgres
         if: always() && steps.install-dependencies.outcome == 'success'
         uses: ./.github/composite-actions/build-modified-postgres
+        with:
+          install_dir: 'psql'
+          code_coverage: 'yes'
 
       - name: Compile ANTLR
         id: compile-antlr
@@ -61,7 +64,7 @@ jobs:
         if: always() && steps.jdbc.outcome == 'failure'
         uses: actions/upload-artifact@v2
         with:
-          name: postgres-log
+          name: postgres-log-jdbc
           path: ~/psql/data/logfile
 
       # The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions
@@ -86,9 +89,43 @@ jobs:
         if: always() && steps.jdbc.outcome == 'failure'
         uses: actions/upload-artifact@v2
         with:
-          name: output-diff.diff
+          name: jdbc-output-diff.diff
           path: test/JDBC/Info/output-diff.diff
 
       - name: Check and upload coredumps
         if: always() && steps.jdbc.outcome == 'failure'
-        uses: ./.github/composite-actions/upload-coredump
+        uses: ./.github/composite-actions/upload-coredump
+
+      - name: Generate Code Coverage
+        id: generate-code-coverage
+        if: always() && steps.jdbc.outcome == 'success'
+        run: |
+          export PG_CONFIG=~/psql/bin/pg_config
+          export PG_SRC=~/work/postgresql_modified_for_babelfish
+          export cmake=$(which cmake)
+          cd contrib
+          for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql  
+          do
+            cd $ext
+            /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info
+            cd ..
+          done
+        shell: bash
+
+      - name: Summarize code coverage
+        id: code-coverage-summary
+        if: always() && steps.generate-code-coverage.outcome == 'success'
+        run: |
+          cd contrib/
+          lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o jdbc-lcov.info
+          lcov --list jdbc-lcov.info
+
+      - name: Upload Coverage Report for Babelfish Extensions
+        if: always() && steps.code-coverage-summary.outcome == 'success'
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-babelfish-extensions-jdbc
+          path: contrib/jdbc-lcov.info
+          retention-days: 1
+
+
diff --git a/.github/workflows/odbc-tests.yml b/.github/workflows/odbc-tests.yml
@@ -1,5 +1,5 @@
 name: ODBC Tests
-on: [push, pull_request]
+on: [workflow_call]
 
 jobs:
   run-babelfish-odbc-tests:
@@ -17,6 +17,9 @@ jobs:
         id: build-modified-postgres
         if: always() && steps.install-dependencies.outcome == 'success'
         uses: ./.github/composite-actions/build-modified-postgres
+        with:
+          install_dir: 'psql'
+          code_coverage: 'yes'
 
       - name: Compile ANTLR
         id: compile-antlr
@@ -42,3 +45,35 @@ jobs:
         id: install-and-run-odbc
         if: steps.install-extensions.outcome == 'success'
         uses: ./.github/composite-actions/install-and-run-odbc
+
+      - name: Generate Code Coverage
+        id: generate-code-coverage
+        if: always() && steps.install-and-run-odbc.outcome == 'success'
+        run: |
+          export PG_CONFIG=~/psql/bin/pg_config
+          export PG_SRC=~/work/postgresql_modified_for_babelfish
+          export cmake=$(which cmake)
+          cd contrib
+          for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql  
+          do
+            cd $ext
+            /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info
+            cd ..
+          done
+        shell: bash
+
+      - name: Summarize code coverage
+        id: code-coverage-summary
+        if: always() && steps.generate-code-coverage.outcome == 'success'
+        run: |
+          cd contrib/
+          lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o odbc-lcov.info
+          lcov --list odbc-lcov.info
+
+      - name: Upload Coverage Report for Babelfish Extensions
+        if: always() && steps.code-coverage-summary.outcome == 'success'
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-babelfish-extensions-odbc
+          path: contrib/odbc-lcov.info
+          retention-days: 1
diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml
@@ -0,0 +1,89 @@
+name: Tests
+on: [push, pull_request]
+
+jobs:
+
+  run-jdbc-tests:
+    name: JDBC Tests
+    uses: ./.github/workflows/jdbc-tests.yml
+
+  run-odbc-tests:
+    name: ODBC Tests
+    uses: ./.github/workflows/odbc-tests.yml
+
+  run-dotnet-tests:
+    name: Dotnet Tests
+    uses: ./.github/workflows/dotnet-tests.yml
+
+  run-python-tests:
+    name: Python Tests
+    uses: ./.github/workflows/python-tests.yml
+
+  run-isolation-tests:
+    name: Isolation Tests
+    uses: ./.github/workflows/isolation-tests.yml
+
+  run-babelfish-code-coverage-for-pull_request:
+    needs: [run-jdbc-tests, run-odbc-tests, run-dotnet-tests, run-python-tests, run-isolation-tests]
+    runs-on: ubuntu-20.04
+    steps:
+
+    - uses: actions/checkout@v2
+      id: checkout
+
+    - name: Install Code Coverage Dependencies
+      id: install-code-coverage-dependencies
+      if: always()
+      run: |
+        sudo apt-get install lcov
+
+    - uses: actions/download-artifact@v3
+      id: download-jdbc-coverage
+      with:
+        name:  coverage-babelfish-extensions-jdbc
+        path: contrib/
+
+    - uses: actions/download-artifact@v3
+      id: download-dotnet-coverage
+      with:
+        name: coverage-babelfish-extensions-dotnet
+        path: contrib/
+
+    - uses: actions/download-artifact@v3
+      id: download-odbc-coverage
+      with:
+        name: coverage-babelfish-extensions-odbc
+        path: contrib/
+
+    - uses: actions/download-artifact@v3
+      id: download-python-coverage
+      with:
+        name: coverage-babelfish-extensions-python
+        path: contrib/
+
+    - uses: actions/download-artifact@v3
+      id: download-isolation-coverage
+      with:
+        name: coverage-babelfish-extensions-isolation
+        path: contrib/
+
+    - name: Generate Overall Code Coverage
+      id: generate-total-code-coverage
+      if: |
+        always() && steps.download-dotnet-coverage.outcome == 'success'
+        && steps.download-jdbc-coverage.outcome == 'success' 
+        && steps.download-odbc-coverage.outcome == 'success' 
+        && steps.download-python-coverage.outcome == 'success'
+        && steps.download-isolation-coverage.outcome == 'success'
+      run: |
+        cd contrib/
+        lcov -a jdbc-lcov.info -a dotnet-lcov.info -a odbc-lcov.info -o lcov.info
+        lcov --list lcov.info
+      shell: bash
+
+    - name: Upload coverage to coveralls
+      if: always() && steps.generate-total-code-coverage.outcome == 'success'
+      uses: coverallsapp/github-action@v2
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        path-to-lcov: $GITHUB_WORKSPACE/contrib/lcov.info