From 5740b181e53dde47580942633801652d9d1207f5 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Wed, 15 Jan 2025 19:42:29 +0000 Subject: [PATCH] temp Signed-off-by: Tanzeel Khan --- .../build-modified-postgres/action.yml | 25 +++++++++++++++++++ .../install-dependencies/action.yml | 9 ------- .../minor-version-upgrade-util/action.yml | 8 ++++-- .../run-jdbc-tests/action.yml | 4 +-- .../run-verify-tests/action.yml | 4 +-- .../save-cache-on-push/action.yml | 21 ++++++++++++++++ .../setup-base-version/action.yml | 8 ++++-- .../setup-new-version/action.yml | 4 +++ .github/workflows/jdbc-tests-db-collation.yml | 4 +-- ...ests-with-non-default-server-collation.yml | 2 +- .../jdbc-tests-with-parallel-query.yml | 2 +- .github/workflows/major-version-upgrade.yml | 2 +- .github/workflows/minor-version-upgrade.yml | 2 +- 13 files changed, 72 insertions(+), 23 deletions(-) create mode 100644 .github/composite-actions/save-cache-on-push/action.yml diff --git a/.github/composite-actions/build-modified-postgres/action.yml b/.github/composite-actions/build-modified-postgres/action.yml index cbd7f086b94..d53f7d2eddd 100644 --- a/.github/composite-actions/build-modified-postgres/action.yml +++ b/.github/composite-actions/build-modified-postgres/action.yml @@ -48,6 +48,31 @@ runs: $GITHUB_WORKSPACE/.github/scripts/clone_engine_repo "$REPOSITORY_OWNER" "$ENGINE_BRANCH" cd postgresql_modified_for_babelfish git rev-parse HEAD + rm -rf ~/.ccache + if [[ ${{inputs.tap_tests}} == "yes" ]]; then + echo "CCACHE_KEY=$(git rev-parse HEAD)-tap" >> $GITHUB_ENV + elif [[ ${{inputs.code_coverage}} == "yes" ]]; then + echo "CCACHE_KEY=$(git rev-parse HEAD)-coverage" >> $GITHUB_ENV + elif [[ ${{inputs.release_mode}} == "yes" ]]; then + echo "CCACHE_KEY=$(git rev-parse HEAD)-release" >> $GITHUB_ENV + else + echo "CCACHE_KEY=$(git rev-parse HEAD)-default" >> $GITHUB_ENV + fi + shell: bash + + - uses: actions/cache/restore@v4 + if: ${{ github.event_name == 'pull_request' }} + with: + path: ~/.ccache + key: ${{ env.CCACHE_KEY }} + + - name: Save cache if cache hit fails in pull requests + if: ${{ github.event_name == 'pull_request' }} && steps.cache.outputs.cache-hit != 'true' + run: echo "SAVE_CCACHE=1" >> $GITHUB_ENV + + - name: Checkout, Build, and Install the Modified PostgreSQL Instance and Run Tests + run: | + cd postgresql_modified_for_babelfish if [[ ${{inputs.tap_tests}} == "yes" ]]; then ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu --enable-tap-tests --with-gssapi elif [[ ${{inputs.code_coverage}} == "yes" ]]; then diff --git a/.github/composite-actions/install-dependencies/action.yml b/.github/composite-actions/install-dependencies/action.yml index de0912e518e..5a271c5e7f5 100644 --- a/.github/composite-actions/install-dependencies/action.yml +++ b/.github/composite-actions/install-dependencies/action.yml @@ -17,12 +17,3 @@ runs: source ~/.bashrc && echo $PATH echo "NOW=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV shell: bash - - - name: Restore ccache - id: cache-compiler - uses: actions/cache@v3 - with: - path: ~/.ccache - key: ccache-${{ runner.os }}-${{ env.NOW }} - restore-keys: | - ccache-${{ runner.os }} diff --git a/.github/composite-actions/minor-version-upgrade-util/action.yml b/.github/composite-actions/minor-version-upgrade-util/action.yml index 519ffa5a70e..8f4d0e6713e 100644 --- a/.github/composite-actions/minor-version-upgrade-util/action.yml +++ b/.github/composite-actions/minor-version-upgrade-util/action.yml @@ -48,6 +48,10 @@ runs: install_dir: ${{ inputs.install_dir }} extension_branch: ${{ inputs.extension_branch }} + - name: Save cache + if: always() && steps.build-extensions-newer == 'success' + uses: ./.github/composite-actions/save-cache-on-push/action.yml + # Not created and used composite action update-extensions here since, in the previous step it has # checked out a branch/tag which may not have the updated update-extension composite action - name: Update extensions @@ -94,11 +98,11 @@ runs: tar_dir="latest" fi export inputFilesPath=upgrade/$tar_dir/verification_cleanup/$base_dir - mvn -B -ntp test + export inputFilesPath=input for filename in $(grep -v "^ignore.*\|^#.*\|^cmd.*\|^all.*\|^$" upgrade/$base_dir/schedule); do sed -i "s/\b$filename[ ]*\b$/$filename-vu-verify\\n$filename-vu-cleanup/g" upgrade/$base_dir/schedule done export scheduleFile=upgrade/$base_dir/schedule - mvn -B -ntp test + shell: bash diff --git a/.github/composite-actions/run-jdbc-tests/action.yml b/.github/composite-actions/run-jdbc-tests/action.yml index 0dad6546e75..6610e173018 100644 --- a/.github/composite-actions/run-jdbc-tests/action.yml +++ b/.github/composite-actions/run-jdbc-tests/action.yml @@ -22,9 +22,9 @@ runs: cd test/JDBC/ if [[ '${{ inputs.migration_mode }}' == 'single-db' ]];then export isSingleDbMode=true - mvn -B -ntp test + unset isSingleDbMode else - mvn -B -ntp test + fi shell: bash diff --git a/.github/composite-actions/run-verify-tests/action.yml b/.github/composite-actions/run-verify-tests/action.yml index 0a058cd73ff..f13cf895910 100644 --- a/.github/composite-actions/run-verify-tests/action.yml +++ b/.github/composite-actions/run-verify-tests/action.yml @@ -61,7 +61,7 @@ runs: export inputFilesPath=upgrade/singledb/verification_cleanup fi - mvn -B -ntp test + export inputFilesPath=input # Skip test BABEL-404 for previous versions while testing dump/restore since it specifies @@ -74,7 +74,7 @@ runs: sed -i "s/\b$filename[ ]*\b$/$filename-vu-verify\\n$filename-vu-cleanup/g" upgrade/$base_dir/schedule done export scheduleFile=upgrade/$base_dir/schedule - mvn -B -ntp test + shell: bash - name: Cleanup babelfish database diff --git a/.github/composite-actions/save-cache-on-push/action.yml b/.github/composite-actions/save-cache-on-push/action.yml new file mode 100644 index 00000000000..1747e20d324 --- /dev/null +++ b/.github/composite-actions/save-cache-on-push/action.yml @@ -0,0 +1,21 @@ +name: 'Save Ccache if push event' + +inputs: + +runs: + using: "composite" + steps: + + - uses: actions/cache/save@v4 + if: ${{ env.CCACHE_KEY != '' && (github.event_name != 'pull_request' || SAVE_CCACHE == 1)}} + with: + path: ~/.ccache + key: ${{ env.CCACHE_KEY }} + + - name: Clean ccache directory and unset env variables + shell: bash + run: | + rm -rf ~/.ccache + echo "CCACHE_KEY=" >> $GITHUB_ENV + echo "SAVE_CCACHE=" >> $GITHUB_ENV + diff --git a/.github/composite-actions/setup-base-version/action.yml b/.github/composite-actions/setup-base-version/action.yml index 66226e6a6ba..2c68aef7007 100644 --- a/.github/composite-actions/setup-base-version/action.yml +++ b/.github/composite-actions/setup-base-version/action.yml @@ -196,7 +196,7 @@ runs: fi export inputFilesPath=upgrade/$base_dir/preparation - mvn -B -ntp test + # Skip test BABEL-404 for previous versions while testing dump/restore since it specifies # contraint column ordering which is expected to fail after fix for BABEL-4888 @@ -216,7 +216,7 @@ runs: sed -i "s/\b$filename[ ]*\b$/$filename-vu-prepare/g" upgrade/$base_dir/schedule done export scheduleFile=upgrade/$base_dir/schedule - mvn -B -ntp test + shell: bash - uses: actions/checkout@v2 @@ -265,3 +265,7 @@ runs: mkdir -p ~/upgrade cp test/python/output/upgrade_validation/* ~/upgrade shell: bash + + - name: Save cache + if: always() && steps.run-dependency-check.outcome == 'success' + uses: ./.github/composite-actions/save-cache-on-push/action.yml diff --git a/.github/composite-actions/setup-new-version/action.yml b/.github/composite-actions/setup-new-version/action.yml index 2138d1cf418..4b564303b7a 100644 --- a/.github/composite-actions/setup-new-version/action.yml +++ b/.github/composite-actions/setup-new-version/action.yml @@ -64,6 +64,10 @@ runs: with: install_dir: ${{ inputs.pg_new_dir }} + - name: Save cache + if: always() && steps.build-postgis-extension == 'success' + uses: ./.github/composite-actions/save-cache-on-push/action.yml + - name: Setup new data directory id: setup-new-datadir if: always() && steps.build-postgis-extension.outcome == 'success' diff --git a/.github/workflows/jdbc-tests-db-collation.yml b/.github/workflows/jdbc-tests-db-collation.yml index 843a615be1f..03e812963d1 100644 --- a/.github/workflows/jdbc-tests-db-collation.yml +++ b/.github/workflows/jdbc-tests-db-collation.yml @@ -65,7 +65,7 @@ jobs: sudo ~/${{env.INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d babelfish_db -U runner -c "select default_collation from sys.babelfish_sysdatabases where name = 'master';" cd test/JDBC/ export isdbCollationMode=true - mvn -B -ntp test + unset isdbCollationMode - name: Start secondary server @@ -96,7 +96,7 @@ jobs: cd test/JDBC/ export isdbCollationMode=true export inputFilesPath=replication - mvn -B -ntp test + unset isdbCollationMode - name: Cleanup babelfish database diff --git a/.github/workflows/jdbc-tests-with-non-default-server-collation.yml b/.github/workflows/jdbc-tests-with-non-default-server-collation.yml index ab266633efa..bd1be496ae5 100644 --- a/.github/workflows/jdbc-tests-with-non-default-server-collation.yml +++ b/.github/workflows/jdbc-tests-with-non-default-server-collation.yml @@ -65,7 +65,7 @@ jobs: cd test/JDBC/ # set env variable serverCollationName to current server collation name export serverCollationName=${{ env.SERVER_COLLATION_NAME }} - mvn -B -ntp test + # reset env variable unset serverCollationName diff --git a/.github/workflows/jdbc-tests-with-parallel-query.yml b/.github/workflows/jdbc-tests-with-parallel-query.yml index 4fea052296d..c9767205cd2 100644 --- a/.github/workflows/jdbc-tests-with-parallel-query.yml +++ b/.github/workflows/jdbc-tests-with-parallel-query.yml @@ -62,7 +62,7 @@ jobs: cd test/JDBC/ # set env variable isParallelQueryMode to true to let jdbc know not to run tests of file parallel_query_jdbc_schedule export isParallelQueryMode=true - mvn -B -ntp test + # reset env variable unset isParallelQueryMode diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml index e98cedce619..1dc427c83dc 100644 --- a/.github/workflows/major-version-upgrade.yml +++ b/.github/workflows/major-version-upgrade.yml @@ -229,7 +229,7 @@ jobs: cd test/JDBC/ # temporarily ignore test BABEL-2086 echo 'ignore#!#BABEL-2086' >> jdbc_schedule - mvn -B -ntp test + - name: Upload Postgres log if: always() && steps.jdbc.outcome == 'failure' diff --git a/.github/workflows/minor-version-upgrade.yml b/.github/workflows/minor-version-upgrade.yml index 3adc33bf2fc..00448be241b 100644 --- a/.github/workflows/minor-version-upgrade.yml +++ b/.github/workflows/minor-version-upgrade.yml @@ -147,7 +147,7 @@ jobs: export PATH=~/${{env.INSTALL_DIR}}/bin:$PATH export PG_SRC=~/work/babelfish_extensions/postgresql_modified_for_babelfish cd test/JDBC/ - mvn -B -ntp test + - name: Upload Log if: always()