Create and use composite actions for code coverage workflow to improve maintainability #279
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code Coverage | |
| on: [push, pull_request] | |
| jobs: | |
| generate-branch-names: | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| matrix: ${{ steps.matrix.outputs.branches }} | |
| steps: | |
| - id: matrix | |
| run: | | |
| if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then | |
| { | |
| echo 'branches<<EOF' | |
| echo "[{ | |
| \"extension_branch\":\"BABEL_3_X_DEV\", | |
| \"engine_branch\":\"BABEL_3_X_DEV__PG_15_X\" | |
| }]" | |
| echo EOF | |
| } >> "$GITHUB_OUTPUT" | |
| else | |
| { | |
| echo 'branches<<EOF' | |
| echo "[{ | |
| \"extension_branch\":\"BABEL_3_X_DEV\", | |
| \"engine_branch\":\"BABEL_3_X_DEV__PG_15_X\" | |
| }, | |
| { | |
| \"extension_branch\":\"BABEL_2_X_DEV\", | |
| \"engine_branch\":\"BABEL_2_X_DEV__PG_14_X\" | |
| }]" | |
| echo EOF | |
| } >> "$GITHUB_OUTPUT" | |
| fi | |
| run-all-tests: | |
| needs: [ generate-branch-names ] | |
| runs-on: ubuntu-20.04 | |
| env: | |
| INSTALL_DIR: psql | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| branch: ${{fromJson(needs.generate-branch-names.outputs.matrix)}} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| id: checkout | |
| # with: | |
| # ref: ${{ matrix.branch.extension_branch }} | |
| - name: Install Dependencies | |
| id: install-dependencies | |
| if: always() | |
| uses: ./.github/composite-actions/install-dependencies | |
| - name: Install Code Coverage Dependencies | |
| id: install-code-coverage-dependencies | |
| if: always() | |
| run: | | |
| sudo apt-get install lcov | |
| - name: Build Modified Postgres | |
| id: build-modified-postgres | |
| if: always() && steps.install-dependencies.outcome == 'success' | |
| uses: ./.github/composite-actions/build-modified-postgres | |
| with: | |
| engine_branch: ${{ matrix.branch.engine_branch }} | |
| install_dir: psql | |
| - name: Compile ANTLR | |
| id: compile-antlr | |
| if: always() && steps.build-modified-postgres.outcome == 'success' | |
| uses: ./.github/composite-actions/compile-antlr | |
| with: | |
| install_dir: 'psql' | |
| - name: Build Extensions | |
| id: build-extensions | |
| if: always() && steps.compile-antlr.outcome == 'success' | |
| uses: ./.github/composite-actions/build-extensions | |
| with: | |
| install_dir: 'psql' | |
| - name: Build tds_fdw Extension | |
| id: build-tds_fdw-extension | |
| if: ${{ startsWith(matrix.branch.extension_branch, 'BABEL_3_') && (steps.build-extensions.outcome == 'success') }} | |
| uses: ./.github/composite-actions/build-tds_fdw-extension | |
| - name: Build PostGIS Extension | |
| id: build-postgis-extension | |
| if: ${{ startsWith(matrix.branch.extension_branch, 'BABEL_3_') && (steps.build-tds_fdw-extension.outcome == 'success') }} | |
| uses: ./.github/composite-actions/build-postgis-extension | |
| - name: Install Extensions | |
| id: install-extensions | |
| if: always() && steps.build-extensions.outcome == 'success' | |
| uses: ./.github/composite-actions/install-extensions | |
| with: | |
| install_dir: 'psql' | |
| - name: Run JDBC Tests | |
| id: jdbc | |
| if: always() && steps.install-extensions.outcome == 'success' | |
| timeout-minutes: 1 | |
| uses: ./.github/composite-actions/run-jdbc-tests | |
| - name: Install MSSQL Tools | |
| id: install-mssql-tools | |
| if: always() && steps.install-extensions.outcome == 'success' | |
| uses: ./.github/composite-actions/install-mssql-tools | |
| - name: Install Dotnet | |
| id: install-dotnet | |
| if: always() && steps.install-extensions.outcome == 'success' | |
| uses: ./.github/composite-actions/install-and-run-dotnet | |
| - name: Install SQL Server ODBC Driver | |
| id: install-sql-server-odbc-driver | |
| if: always() && steps.install-extensions.outcome == 'success' | |
| run: | | |
| cd ~ | |
| sudo apt-get install msodbcsql17 | |
| - name: Install unixODBC Driver | |
| id: install-unix-odbc-driver | |
| if: always() && steps.install-extensions.outcome == 'success' | |
| run: | | |
| cd ~ | |
| wget http://www.unixodbc.org/unixODBC-2.3.11.tar.gz | |
| gunzip unixODBC*.tar.gz | |
| tar xvf unixODBC*.tar | |
| cd unixODBC-2.3.11 | |
| ./configure | |
| make | |
| sudo make install | |
| - name: Install psqlODBC Driver | |
| id: install-psql-odbc-driver | |
| if: always() && steps.install-extensions.outcome == 'success' && steps.install-unix-odbc-driver.outcome=='success' | |
| run: | | |
| cd ~ | |
| wget https://ftp.postgresql.org/pub/odbc/versions/src/psqlodbc-16.00.0000.tar.gz | |
| tar -zxvf psqlodbc-16.00.0000.tar.gz | |
| cd psqlodbc-16.00.0000 | |
| ./configure | |
| sudo make | |
| sudo make install | |
| echo '[ODBC_Driver_16_for_PostgreSQL]' | sudo tee -a /etc/odbcinst.ini > /dev/null | |
| echo 'Description=ODBC Driver 16 for PostgreSQL Server' | sudo tee -a /etc/odbcinst.ini > /dev/null | |
| echo 'Driver=/usr/local/lib/psqlodbcw.so' | sudo tee -a /etc/odbcinst.ini > /dev/null | |
| echo 'UsageCount=1' | sudo tee -a /etc/odbcinst.ini > /dev/null | |
| - name: Run ODBC Tests | |
| if: always() && steps.install-sql-server-odbc-driver.outcome == 'success' && steps.install-psql-odbc-driver.outcome == 'success' | |
| run: | | |
| cd test/odbc | |
| cmake -S . -B build | |
| cmake --build build | |
| MSSQL_ODBC_DRIVER_NAME="ODBC Driver 17 for SQL Server" \ | |
| MSSQL_BABEL_DB_SERVER=localhost \ | |
| MSSQL_BABEL_DB_PORT=1433 \ | |
| MSSQL_BABEL_DB_USER=jdbc_user \ | |
| MSSQL_BABEL_DB_PASSWORD=12345678 \ | |
| MSSQL_BABEL_DB_NAME=master \ | |
| PSQL_ODBC_DRIVER_NAME=ODBC_Driver_16_for_PostgreSQL \ | |
| PSQL_BABEL_DB_SERVER=localhost \ | |
| PSQL_BABEL_DB_PORT=5432 \ | |
| PSQL_BABEL_DB_USER=jdbc_user \ | |
| PSQL_BABEL_DB_PASSWORD=12345678 \ | |
| PSQL_BABEL_DB_NAME=jdbc_testdb \ | |
| ./build/main | |
| - name: Install Python | |
| id: install-python | |
| if: always() && steps.install-extensions.outcome == 'success' | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.7 | |
| - name: Configure Python Environment | |
| id: configure-python-environment | |
| if: always() && steps.install-python.outcome == 'success' | |
| run: | | |
| cd ~ | |
| curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list | |
| cd ~/work/babelfish_extensions/babelfish_extensions/test/python | |
| mkdir sqltoolsservice | |
| cd sqltoolsservice | |
| wget https://github.com/microsoft/sqltoolsservice/releases/download/4.4.0.12/Microsoft.SqlTools.ServiceLayer-rhel-x64-net6.0.tar.gz && tar -xzvf Microsoft.SqlTools.ServiceLayer-rhel-x64-net6.0.tar.gz | |
| cd ../ | |
| sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 python3-dev | |
| pip3 install pyodbc==4.0.35 pymssql pytest pytest-xdist | |
| - name: Drop and re-create Babelfish database | |
| id: re-install-extensions | |
| if: always() && steps.configure-python-environment.outcome == 'success' | |
| run: | | |
| sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -f .github/scripts/cleanup_babelfish_database.sql | |
| sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode="single_db" -f .github/scripts/create_extension.sql | |
| sqlcmd -S localhost -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO" | |
| - name: Run Python Tests | |
| if: always() && steps.re-install-extensions.outcome == 'success' | |
| run: | | |
| cd test/python | |
| compareWithFile=true \ | |
| driver=pyodbc \ | |
| runInParallel=false \ | |
| testName=all \ | |
| provider="ODBC Driver 17 for SQL Server" \ | |
| fileGenerator_URL=localhost \ | |
| fileGenerator_port=1433 \ | |
| fileGenerator_databaseName=master \ | |
| fileGenerator_user=jdbc_user \ | |
| fileGenerator_password=12345678 \ | |
| pytest -s --tb=long -q . | |
| - name: Generate code coverage HTML report | |
| id: code-coverage | |
| if: always() | |
| run: | | |
| export PG_CONFIG=~/psql/bin/pg_config | |
| export PG_SRC=~/work/postgresql_modified_for_babelfish | |
| export cmake=$(which cmake) | |
| cd contrib | |
| for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql | |
| do | |
| cd $ext | |
| /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -i -d . -d ./ -o lcov_base.info | |
| /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info | |
| rm -rf coverage | |
| /usr/bin/genhtml -q --legend -o coverage --title='$ext' --ignore-errors source --num-spaces=4 lcov_base.info lcov_test.info | |
| touch coverage-html-stamp | |
| cd .. | |
| done | |
| shell: bash | |
| - name: Summarize code coverage | |
| id: code-coverage-summary | |
| if: always() | |
| run: | | |
| cd contrib/ | |
| lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o lcov.info | |
| lcov --list lcov.info | |
| - name: Upload Coverage Report for babelfishpg_tsql extension | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage_tsql_${{ matrix.branch.extension_branch }} | |
| path: contrib/babelfishpg_tsql/coverage/ | |
| - name: Upload Coverage Report for babelfishpg_tds extension | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage_tds_${{ matrix.branch.extension_branch }} | |
| path: contrib/babelfishpg_tds/coverage/ | |
| - name: Upload Coverage Report for babelfishpg_common extension | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage_common_${{ matrix.branch.extension_branch }} | |
| path: contrib/babelfishpg_common/coverage/ | |
| - name: Upload Coverage Report for babelfishpg_money extension | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage_money_${{ matrix.branch.extension_branch }} | |
| path: contrib/babelfishpg_money/coverage/ | |
| - name: Download CSV report from previous run | |
| if: (github.event_name == 'schedule') | |
| uses: dawidd6/action-download-artifact@v2 | |
| with: | |
| name: csv_${{ matrix.branch.extension_branch }} | |
| path: contrib/ | |
| search_artifacts: true | |
| if_no_artifact_found: warn | |
| - name: Add latest coverage numbers to CSV file | |
| if: (github.event_name == 'schedule') | |
| run: | | |
| cd contrib/ | |
| paste -s -d, <(date +"%m/%d/%Y %H:%M:%S";lcov --summary lcov.info | grep -Po "[0-9]+\.[0-9]*") >> ${{ matrix.branch.extension_branch }}.csv | |
| shell: bash | |
| - name: Upload CSV report with latest coverage numbers | |
| if: (github.event_name == 'schedule') | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: csv_${{ matrix.branch.extension_branch }} | |
| path: contrib/${{ matrix.branch.extension_branch }}.csv |