From 57209666803c97e62ec466515f133546763042cf Mon Sep 17 00:00:00 2001 From: Sharu Goel <30777678+thephantomthief@users.noreply.github.com> Date: Thu, 4 Jan 2024 17:57:45 +0530 Subject: [PATCH] PR Code Coverage with Coveralls (#2205) urrently we run code coverage as scheduled job on a daily basis where all the tests are run one after the another. We aim to generate coverage for pull request as well. But running all the test frameworks sequentially takes too long. So we create a new workflow which will trigger all the tests in parallel and merge the individual code coverage reports into one overall code coverage report. Specifically we are merging the .info files generated by lcov. These files contains the count of times each line / function has been hit & the relative path for this line / function in source code. So as long as we use the same source code, merging these files will be okay. The consolidated lcov.info is uploaded to Coveralls - A web-based code coverage service that provided useful insights about the how well the newly introduced code diff in a pull request is covered by the test suites. Task: BABEL-4602 Co-authored-by: Tanzeel Khan tzlkhan@amazon.com, Sharu Goel goelshar@amazon.com Signed-off-by: Sharu Goel goelshar@amazon.com --- .../build-modified-postgres/action.yml | 6 +- .../install-dependencies/action.yml | 1 + .github/workflows/code-coverage.yml | 6 -- .github/workflows/dotnet-tests.yml | 37 +++++++- .github/workflows/isolation-tests.yml | 38 +++++++- .github/workflows/jdbc-tests.yml | 45 +++++++++- .github/workflows/odbc-tests.yml | 37 +++++++- .github/workflows/pr-code-coverage.yml | 89 +++++++++++++++++++ .github/workflows/python-tests.yml | 37 +++++++- 9 files changed, 278 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/pr-code-coverage.yml diff --git a/.github/composite-actions/build-modified-postgres/action.yml b/.github/composite-actions/build-modified-postgres/action.yml index 4310a40762..f3d14db6d3 100644 --- a/.github/composite-actions/build-modified-postgres/action.yml +++ b/.github/composite-actions/build-modified-postgres/action.yml @@ -51,12 +51,10 @@ runs: ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu --enable-tap-tests --with-gssapi elif [[ ${{inputs.code_coverage}} == "yes" ]]; then ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-coverage --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu - else - if [[ ${{inputs.release_mode}} == "yes" ]]; then + elif [[ ${{inputs.release_mode}} == "yes" ]]; then ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 CFLAGS="-ggdb -O2" --with-libxml --with-uuid=ossp --with-icu - else + else ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu - fi fi make -j 4 2>error.txt make install diff --git a/.github/composite-actions/install-dependencies/action.yml b/.github/composite-actions/install-dependencies/action.yml index 658d6ad489..c63c54e031 100644 --- a/.github/composite-actions/install-dependencies/action.yml +++ b/.github/composite-actions/install-dependencies/action.yml @@ -11,6 +11,7 @@ runs: sudo apt-get update --fix-missing -y sudo apt-get install uuid-dev openjdk-8-jre libicu-dev libxml2-dev openssl libssl-dev python3-dev libossp-uuid-dev libpq-dev cmake pkg-config g++ build-essential bison mssql-tools unixodbc-dev libsybdb5 freetds-dev freetds-common gdal-bin libgdal-dev libgeos-dev gdb sudo apt install -y ccache + sudo apt-get install lcov sudo /usr/sbin/update-ccache-symlinks echo 'export PATH="/usr/lib/ccache:$PATH"' | tee -a ~/.bashrc source ~/.bashrc && echo $PATH diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 92bc9bde59..74380efec6 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -17,12 +17,6 @@ jobs: if: always() uses: ./.github/composite-actions/install-dependencies - - name: Install Code Coverage Dependencies - id: install-code-coverage-dependencies - if: always() - run: | - sudo apt-get install lcov - - name: Build Modified Postgres id: build-modified-postgres if: always() && steps.install-dependencies.outcome == 'success' diff --git a/.github/workflows/dotnet-tests.yml b/.github/workflows/dotnet-tests.yml index 0d47f5694f..32d3936118 100644 --- a/.github/workflows/dotnet-tests.yml +++ b/.github/workflows/dotnet-tests.yml @@ -1,5 +1,5 @@ name: Dotnet Framework Tests -on: [push, pull_request] +on: [workflow_call] jobs: run-babelfish-dotnet-tests: @@ -17,6 +17,9 @@ jobs: id: build-modified-postgres if: always() && steps.install-dependencies.outcome == 'success' uses: ./.github/composite-actions/build-modified-postgres + with: + install_dir: 'psql' + code_coverage: 'yes' - name: Compile ANTLR id: compile-antlr @@ -42,3 +45,35 @@ jobs: id: run-dotnet-tests if: always() && steps.install-extensions.outcome == 'success' uses: ./.github/composite-actions/install-and-run-dotnet + + - name: Generate Code Coverage + id: generate-code-coverage + if: always() && steps.run-dotnet-tests.outcome == 'success' + run: | + export PG_CONFIG=~/psql/bin/pg_config + export PG_SRC=~/work/postgresql_modified_for_babelfish + export cmake=$(which cmake) + cd contrib + for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql + do + cd $ext + /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info + cd .. + done + shell: bash + + - name: Summarize code coverage + id: code-coverage-summary + if: always() && steps.generate-code-coverage.outcome == 'success' + run: | + cd contrib/ + lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o dotnet-lcov.info + lcov --list dotnet-lcov.info + + - name: Upload Coverage Report for Babelfish Extensions + if: always() && steps.code-coverage-summary.outcome == 'success' + uses: actions/upload-artifact@v3 + with: + name: coverage-babelfish-extensions-dotnet + path: contrib/dotnet-lcov.info + retention-days: 1 diff --git a/.github/workflows/isolation-tests.yml b/.github/workflows/isolation-tests.yml index f3c1aa91cd..abbc98a984 100644 --- a/.github/workflows/isolation-tests.yml +++ b/.github/workflows/isolation-tests.yml @@ -1,5 +1,5 @@ name: Babelfish Smoke Tests -on: [push, pull_request] +on: [workflow_call] jobs: isolation-tests: @@ -18,6 +18,9 @@ jobs: id: build-modified-postgres if: always() && steps.install-dependencies.outcome == 'success' uses: ./.github/composite-actions/build-modified-postgres + with: + install_dir: 'psql' + code_coverage: 'yes' - name: Compile ANTLR id: compile-antlr @@ -58,6 +61,7 @@ jobs: java -Xmx500M -cp /usr/local/lib/antlr-4.9.3-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener - name: Run Isolation tests + id: run-isolation-tests run: | cd test/python compareWithFile=true \ @@ -74,3 +78,35 @@ jobs: runIsolationTests=true \ stepTimeLimit=30 \ pytest -s --tb=long -q . + + - name: Generate Code Coverage + id: generate-code-coverage + if: always() && steps.run-isolation-tests.outcome == 'success' + run: | + export PG_CONFIG=~/psql/bin/pg_config + export PG_SRC=~/work/postgresql_modified_for_babelfish + export cmake=$(which cmake) + cd contrib + for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql + do + cd $ext + /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info + cd .. + done + shell: bash + + - name: Summarize code coverage + id: code-coverage-summary + if: always() && steps.generate-code-coverage.outcome == 'success' + run: | + cd contrib/ + lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o isolation-lcov.info + lcov --list isolation-lcov.info + + - name: Upload Coverage Report for Babelfish Extensions + if: always() && steps.code-coverage-summary.outcome == 'success' + uses: actions/upload-artifact@v3 + with: + name: coverage-babelfish-extensions-isolation + path: contrib/isolation-lcov.info + retention-days: 1 diff --git a/.github/workflows/jdbc-tests.yml b/.github/workflows/jdbc-tests.yml index cd63c0cb51..24a3652890 100644 --- a/.github/workflows/jdbc-tests.yml +++ b/.github/workflows/jdbc-tests.yml @@ -1,5 +1,5 @@ name: JDBC Tests -on: [push, pull_request] +on: [workflow_call] jobs: run-babelfish-jdbc-tests: @@ -19,6 +19,9 @@ jobs: id: build-modified-postgres if: always() && steps.install-dependencies.outcome == 'success' uses: ./.github/composite-actions/build-modified-postgres + with: + install_dir: 'psql' + code_coverage: 'yes' - name: Compile ANTLR id: compile-antlr @@ -61,7 +64,7 @@ jobs: if: always() && steps.jdbc.outcome == 'failure' uses: actions/upload-artifact@v2 with: - name: postgres-log + name: postgres-log-jdbc path: ~/psql/data/logfile # The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions @@ -86,9 +89,43 @@ jobs: if: always() && steps.jdbc.outcome == 'failure' uses: actions/upload-artifact@v2 with: - name: output-diff.diff + name: jdbc-output-diff.diff path: test/JDBC/Info/output-diff.diff - name: Check and upload coredumps if: always() && steps.jdbc.outcome == 'failure' - uses: ./.github/composite-actions/upload-coredump \ No newline at end of file + uses: ./.github/composite-actions/upload-coredump + + - name: Generate Code Coverage + id: generate-code-coverage + if: always() && steps.jdbc.outcome == 'success' + run: | + export PG_CONFIG=~/psql/bin/pg_config + export PG_SRC=~/work/postgresql_modified_for_babelfish + export cmake=$(which cmake) + cd contrib + for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql + do + cd $ext + /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info + cd .. + done + shell: bash + + - name: Summarize code coverage + id: code-coverage-summary + if: always() && steps.generate-code-coverage.outcome == 'success' + run: | + cd contrib/ + lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o jdbc-lcov.info + lcov --list jdbc-lcov.info + + - name: Upload Coverage Report for Babelfish Extensions + if: always() && steps.code-coverage-summary.outcome == 'success' + uses: actions/upload-artifact@v3 + with: + name: coverage-babelfish-extensions-jdbc + path: contrib/jdbc-lcov.info + retention-days: 1 + + diff --git a/.github/workflows/odbc-tests.yml b/.github/workflows/odbc-tests.yml index 388f57fdd6..83da6ec34b 100644 --- a/.github/workflows/odbc-tests.yml +++ b/.github/workflows/odbc-tests.yml @@ -1,5 +1,5 @@ name: ODBC Tests -on: [push, pull_request] +on: [workflow_call] jobs: run-babelfish-odbc-tests: @@ -17,6 +17,9 @@ jobs: id: build-modified-postgres if: always() && steps.install-dependencies.outcome == 'success' uses: ./.github/composite-actions/build-modified-postgres + with: + install_dir: 'psql' + code_coverage: 'yes' - name: Compile ANTLR id: compile-antlr @@ -42,3 +45,35 @@ jobs: id: install-and-run-odbc if: steps.install-extensions.outcome == 'success' uses: ./.github/composite-actions/install-and-run-odbc + + - name: Generate Code Coverage + id: generate-code-coverage + if: always() && steps.install-and-run-odbc.outcome == 'success' + run: | + export PG_CONFIG=~/psql/bin/pg_config + export PG_SRC=~/work/postgresql_modified_for_babelfish + export cmake=$(which cmake) + cd contrib + for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql + do + cd $ext + /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info + cd .. + done + shell: bash + + - name: Summarize code coverage + id: code-coverage-summary + if: always() && steps.generate-code-coverage.outcome == 'success' + run: | + cd contrib/ + lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o odbc-lcov.info + lcov --list odbc-lcov.info + + - name: Upload Coverage Report for Babelfish Extensions + if: always() && steps.code-coverage-summary.outcome == 'success' + uses: actions/upload-artifact@v3 + with: + name: coverage-babelfish-extensions-odbc + path: contrib/odbc-lcov.info + retention-days: 1 diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml new file mode 100644 index 0000000000..476c2d7eb4 --- /dev/null +++ b/.github/workflows/pr-code-coverage.yml @@ -0,0 +1,89 @@ +name: Tests +on: [push, pull_request] + +jobs: + + run-jdbc-tests: + name: JDBC Tests + uses: ./.github/workflows/jdbc-tests.yml + + run-odbc-tests: + name: ODBC Tests + uses: ./.github/workflows/odbc-tests.yml + + run-dotnet-tests: + name: Dotnet Tests + uses: ./.github/workflows/dotnet-tests.yml + + run-python-tests: + name: Python Tests + uses: ./.github/workflows/python-tests.yml + + run-isolation-tests: + name: Isolation Tests + uses: ./.github/workflows/isolation-tests.yml + + run-babelfish-code-coverage-for-pull_request: + needs: [run-jdbc-tests, run-odbc-tests, run-dotnet-tests, run-python-tests, run-isolation-tests] + runs-on: ubuntu-20.04 + steps: + + - uses: actions/checkout@v2 + id: checkout + + - name: Install Code Coverage Dependencies + id: install-code-coverage-dependencies + if: always() + run: | + sudo apt-get install lcov + + - uses: actions/download-artifact@v3 + id: download-jdbc-coverage + with: + name:  coverage-babelfish-extensions-jdbc + path: contrib/ + + - uses: actions/download-artifact@v3 + id: download-dotnet-coverage + with: + name: coverage-babelfish-extensions-dotnet + path: contrib/ + + - uses: actions/download-artifact@v3 + id: download-odbc-coverage + with: + name: coverage-babelfish-extensions-odbc + path: contrib/ + + - uses: actions/download-artifact@v3 + id: download-python-coverage + with: + name: coverage-babelfish-extensions-python + path: contrib/ + + - uses: actions/download-artifact@v3 + id: download-isolation-coverage + with: + name: coverage-babelfish-extensions-isolation + path: contrib/ + + - name: Generate Overall Code Coverage + id: generate-total-code-coverage + if: | + always() && steps.download-dotnet-coverage.outcome == 'success' + && steps.download-jdbc-coverage.outcome == 'success' + && steps.download-odbc-coverage.outcome == 'success' + && steps.download-python-coverage.outcome == 'success' + && steps.download-isolation-coverage.outcome == 'success' + run: | + cd contrib/ + lcov -a jdbc-lcov.info -a dotnet-lcov.info -a odbc-lcov.info -o lcov.info + lcov --list lcov.info + shell: bash + + - name: Upload coverage to coveralls + if: always() && steps.generate-total-code-coverage.outcome == 'success' + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: $GITHUB_WORKSPACE/contrib/lcov.info diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index ffcd9c74ee..f9e1c8d5cd 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -1,5 +1,5 @@ name: Python Tests -on: [push, pull_request] +on: [workflow_call] jobs: run-babelfish-python-tests: @@ -17,6 +17,9 @@ jobs: id: build-modified-postgres if: always() && steps.install-dependencies.outcome == 'success' uses: ./.github/composite-actions/build-modified-postgres + with: + install_dir: 'psql' + code_coverage: 'yes' - name: Compile ANTLR id: compile-antlr @@ -42,3 +45,35 @@ jobs: id: run-python-tests if: always() && steps.install-extensions.outcome == 'success' uses: ./.github/composite-actions/install-and-run-python + + - name: Generate Code Coverage + id: generate-code-coverage + if: always() && steps.run-python-tests.outcome == 'success' + run: | + export PG_CONFIG=~/psql/bin/pg_config + export PG_SRC=~/work/postgresql_modified_for_babelfish + export cmake=$(which cmake) + cd contrib + for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql + do + cd $ext + /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info + cd .. + done + shell: bash + + - name: Summarize code coverage + id: code-coverage-summary + if: always() && steps.generate-code-coverage.outcome == 'success' + run: | + cd contrib/ + lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o python-lcov.info + lcov --list python-lcov.info + + - name: Upload Coverage Report for Babelfish Extensions + if: always() && steps.code-coverage-summary.outcome == 'success' + uses: actions/upload-artifact@v3 + with: + name: coverage-babelfish-extensions-python + path: contrib/python-lcov.info + retention-days: 1