[OSS-ONLY] Implementing hashcode and mode based caching

.github/composite-actions/build-modified-postgres/action.yml

@@ -24,11 +24,11 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Checkout, Build, and Install the Modified PostgreSQL Instance and Run Tests
+    - name: Checkout Modified PostgreSQL for Babelfish
       run: |
         cd ..
         rm -rf postgresql_modified_for_babelfish
-        
+
         if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then
           if [[ ${{inputs.engine_branch}} == "latest" ]]; then
             ENGINE_BRANCH=$GITHUB_HEAD_REF
@@ -44,10 +44,47 @@ runs:
           fi
           REPOSITORY_OWNER=$GITHUB_REPOSITORY_OWNER
         fi
-
         $GITHUB_WORKSPACE/.github/scripts/clone_engine_repo "$REPOSITORY_OWNER" "$ENGINE_BRANCH"
         cd postgresql_modified_for_babelfish
-        git rev-parse HEAD
+        rm -rf ~/.ccache
+        if [[ ${{inputs.tap_tests}} == "yes" ]]; then
+          echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-tapTest" >> $GITHUB_ENV
+        elif [[ ${{inputs.code_coverage}} == "yes" ]]; then
+          echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-coverage" >> $GITHUB_ENV
+        elif [[ ${{inputs.release_mode}} == "yes" ]]; then
+          echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-release" >> $GITHUB_ENV
+        else
+          echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-default" >> $GITHUB_ENV
+        fi
+      shell: bash
+
+    - uses: actions/cache/restore@v4
+      id: restore-ccache
+      if: ${{ github.event_name == 'pull_request' }} || ${{ inputs.code_coverage == 'yes' }}
+      with:
+        path: 
+          ~/.ccache
+        key: 
+          random-random
+        restore-keys: 
+          ccache-${{ env.CRESTORE_KEY }}
+
+    - name: Save cache if cache hit fails in pull requests
+      if: ${{ github.event_name == 'pull_request' }}
+      run: |
+        if [[ '${{ steps.restore-ccache.outputs.cache-matched-key }}' == '' ]]; then
+          echo "SAVE_CCACHE=1" >> $GITHUB_ENV
+        elif [[ ${{ 'steps.restore-ccache.outputs.cache-matched-key' }} != '' ]]; then
+          echo "SAVE_CCACHE=" >> $GITHUB_ENV
+        fi
+      shell: bash
+
+    - name: Build and Install the Modified PostgreSQL Instance and Run Tests
+      run: |
+        echo "${{env.SAVE_CCACHE}}"
+        echo "${{ steps.restore-ccache.outputs.cache-matched-key }}"
+        cd ..
+        cd postgresql_modified_for_babelfish
         if [[ ${{inputs.tap_tests}} == "yes" ]]; then
           ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu --enable-tap-tests --with-gssapi
         elif [[ ${{inputs.code_coverage}} == "yes" ]]; then

.github/composite-actions/dump-restore-util/action.yml

@@ -327,6 +327,10 @@ runs:
         sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO"
       shell: bash
 
+    - name: Save cache
+      if: always()
+      uses: ./.github/composite-actions/save-ccache
+
     - name: Run Verify Tests
       if: always() && steps.run-pg_dump-restore.outcome == 'success' && inputs.is_final_ver == 'true'
       uses: ./.github/composite-actions/run-verify-tests

.github/composite-actions/install-dependencies/action.yml

@@ -15,14 +15,4 @@ runs:
         sudo /usr/sbin/update-ccache-symlinks
         echo 'export PATH="/usr/lib/ccache:$PATH"' | tee -a ~/.bashrc
         source ~/.bashrc && echo $PATH
-        echo "NOW=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
       shell: bash
-
-    - name: Restore ccache
-      id: cache-compiler
-      uses: actions/cache@v3
-      with:
-        path: ~/.ccache
-        key: ccache-${{ runner.os }}-${{ env.NOW }}
-        restore-keys: |
-          ccache-${{ runner.os }}

.github/composite-actions/minor-version-upgrade-util/action.yml

@@ -48,6 +48,10 @@ runs:
         install_dir: ${{ inputs.install_dir }}
         extension_branch: ${{ inputs.extension_branch }}
 
+    - name: Save cache
+      if: always() && steps.build-extensions-newer == 'success'
+      uses: ./.github/composite-actions/save-ccache
+
     # Not created and used composite action update-extensions here since, in the previous step it has 
     # checked out a branch/tag which may not have the updated update-extension composite action
     - name: Update extensions

.github/composite-actions/save-ccache/action.yml

@@ -0,0 +1,39 @@
+name: 'Save ccache on push'
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Setup new cache key
+      if: always()
+      run: |
+        echo "CCACHE_KEY=${{env.CRESTORE_KEY}}-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+        echo "Event : ${{ github.event_name }}"
+        echo "Save ccache : ${{ env.SAVE_CCACHE }}"
+        if [[ ${{ github.event_name != 'pull_request' || env.SAVE_CCACHE == 1 }} == true ]]; then
+          echo "Cache to be Saved"
+          echo "SAVE_CACHE_HERE=1" >> $GITHUB_ENV
+        else
+          echo "Cache shouldn't be Saved"
+          echo "SAVE_CACHE_HERE=0" >> $GITHUB_ENV
+        fi
+      shell: bash
+
+    - name: Save ccache
+      uses: actions/cache/save@v4
+      if: env.SAVE_CACHE_HERE == 1
+      with:
+        path: 
+          ~/.ccache
+        key: 
+          ccache-${{ env.CCACHE_KEY }}
+
+    - name: Clean ccache directory and unset env variables
+      if: always()
+      shell: bash
+      run: |
+        rm -rf ~/.ccache
+        echo "CCACHE_KEY=" >> $GITHUB_ENV
+        echo "SAVE_CCACHE=" >> $GITHUB_ENV
+        echo "SAVE_CACHE_HERE=" >> $GITHUB_ENV
+        echo "CRESTORE_KEY=" >> $GITHUB_ENV

.github/composite-actions/setup-base-version/action.yml

@@ -221,6 +221,10 @@ runs:
 
     - uses: actions/checkout@v2
 
+    - name: Save cache
+      if: always() && steps.jdbc-upgrade-tests.outcome == 'success'
+      uses: ./.github/composite-actions/save-ccache
+
     - name: Install Python
       id: install-python
       if: ${{ matrix.upgrade-path.path[0] == 'source_latest' && steps.jdbc-upgrade-tests.outcome == 'success' }}

.github/composite-actions/setup-new-version/action.yml

@@ -64,6 +64,10 @@ runs:
       with:
         install_dir: ${{ inputs.pg_new_dir }}
 
+    - name: Save cache
+      if: always() && steps.build-postgis-extension == 'success'
+      uses: ./.github/composite-actions/save-ccache
+
     - name: Setup new data directory
       id: setup-new-datadir
       if: always() && steps.build-postgis-extension.outcome == 'success'

.github/composite-actions/upload-coredump/action.yml

@@ -14,7 +14,7 @@ runs:
 
 
     - name: Upload Coredumps
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: always()
       with:
         name: coredumps

.github/workflows/code-coverage.yml

@@ -119,28 +119,28 @@ jobs:
           lcov --list lcov.info
       - name: Upload Coverage Report for babelfishpg_tsql extension
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_tsql_${{github.ref_name}}
           path: contrib/babelfishpg_tsql/coverage/
 
       - name: Upload Coverage Report for babelfishpg_tds extension
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_tds_${{github.ref_name}}
           path: contrib/babelfishpg_tds/coverage/
 
       - name: Upload Coverage Report for babelfishpg_common extension
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_common_${{github.ref_name}}
           path: contrib/babelfishpg_common/coverage/
 
       - name: Upload Coverage Report for babelfishpg_money extension
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_money_${{github.ref_name}}
           path: contrib/babelfishpg_money/coverage/
@@ -163,7 +163,7 @@ jobs:
 
       - name: Upload CSV report with latest coverage numbers
         if: (github.event_name == 'schedule')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: csv_${{github.ref_name}}
           path: contrib/${{github.ref_name}}.csv

.github/workflows/dotnet-tests.yml

@@ -40,6 +40,10 @@ jobs:
         id: install-extensions
         if: always() && steps.build-postgis-extension.outcome == 'success'
         uses: ./.github/composite-actions/install-extensions
+
+      - name: Save cache
+        if: always() && steps.install-extensions.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
 
       - name: Run Dotnet Tests
         id: run-dotnet-tests
@@ -72,7 +76,7 @@ jobs:
 
       - name: Upload Coverage Report for Babelfish Extensions
         if: always() && steps.code-coverage-summary.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-babelfish-extensions-dotnet
           path: contrib/dotnet-lcov.info

.github/workflows/isolation-tests.yml

@@ -105,7 +105,7 @@ jobs:
 
       - name: Upload Coverage Report for Babelfish Extensions
         if: always() && steps.code-coverage-summary.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-babelfish-extensions-isolation
           path: contrib/isolation-lcov.info

.github/workflows/jdbc-tests-single-db-mode.yml

@@ -102,6 +102,10 @@ jobs:
             ~/psql/data/logfile
             ~/psql/data_5433/logfile
 
+      - name: Save cache
+        if: always() && steps.replication.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
+
       # The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions
       - name: Rename Test Summary Files
         id: test-file-rename

.github/workflows/jdbc-tests.yml

@@ -167,7 +167,7 @@ jobs:
 
       - name: Upload Coverage Report for Babelfish Extensions
         if: always() && steps.code-coverage-summary.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-babelfish-extensions-jdbc
           path: contrib/jdbc-lcov.info

.github/workflows/major-version-upgrade.yml

@@ -20,18 +20,11 @@ jobs:
 
       - name: Build Modified Postgres using ${{env.ENGINE_BRANCH_FROM}}
         id: build-modified-postgres-old
+        uses: ./.github/composite-actions/build-modified-postgres
         if: always() && steps.install-dependencies.outcome == 'success'
-        run: |
-          cd ..
-          git clone --branch ${{env.ENGINE_BRANCH_FROM}} https://github.com/babelfish-for-postgresql/postgresql_modified_for_babelfish.git
-          cd postgresql_modified_for_babelfish
-          ./configure --prefix=$HOME/${{env.OLD_INSTALL_DIR}} --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu
-          make clean
-          make -j 4
-          make install
-          make check
-          cd contrib && make && sudo make install
-        shell: bash
+        with:
+          engine_branch: ${{ env.ENGINE_BRANCH_FROM }}
+          install_dir: ${{ env.OLD_INSTALL_DIR }}
 
       - name: Compile ANTLR
         id: compile-antlr
@@ -130,6 +123,10 @@ jobs:
 
       - uses: actions/checkout@v2
 
+      - name: Save cache
+        if: always() && steps.build-extensions-old.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
+
       - name: Build Modified Postgres using latest version
         id: build-modified-postgres-new
         if: always() && steps.install-extensions-old.outcome == 'success'
@@ -172,6 +169,10 @@ jobs:
         uses: ./.github/composite-actions/build-postgis-extension
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}
+
+      - name: Save cache
+        if: always() && steps.build-postgis-extension.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
 
       - name: Setup new data directory
         id: setup-new-datadir

.github/workflows/minor-version-upgrade.yml

@@ -111,6 +111,10 @@ jobs:
 
       - uses: actions/checkout@v2
 
+      - name: Save cache
+        if: always() && steps.build-extensions-older.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
+
       - name: Build and run tests for Postgres engine using latest engine
         id: build-modified-postgres-newer
         if: always() && steps.install-extensions-older.outcome == 'success'

.github/workflows/odbc-tests.yml

@@ -72,7 +72,7 @@ jobs:
 
       - name: Upload Coverage Report for Babelfish Extensions
         if: always() && steps.code-coverage-summary.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-babelfish-extensions-odbc
           path: contrib/odbc-lcov.info

.github/workflows/pr-code-coverage.yml

@@ -37,31 +37,31 @@ jobs:
       run: |
         sudo apt-get install lcov
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       id: download-jdbc-coverage
       with:
         name:  coverage-babelfish-extensions-jdbc
         path: contrib/
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       id: download-dotnet-coverage
       with:
         name: coverage-babelfish-extensions-dotnet
         path: contrib/
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       id: download-odbc-coverage
       with:
         name: coverage-babelfish-extensions-odbc
         path: contrib/
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       id: download-python-coverage
       with:
         name: coverage-babelfish-extensions-python
         path: contrib/
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       id: download-isolation-coverage
       with:
         name: coverage-babelfish-extensions-isolation

.github/workflows/python-tests.yml

@@ -72,7 +72,7 @@ jobs:
 
       - name: Upload Coverage Report for Babelfish Extensions
         if: always() && steps.code-coverage-summary.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-babelfish-extensions-python
           path: contrib/python-lcov.info