Use architecture in save/restore cache keys, move to newer runners, add

mac arm runner Bump to ubuntu 22.04 and macOS-12 runners Add new macOS 14 M1 runner for arm wheels Signed-off-by: Tim Paine <[email protected]>
Point72 · Feb 4, 2024 · 9afbc62 · 9afbc62
1 parent 2acdade
commit 9afbc62
Show file tree

Hide file tree

Showing 5 changed files with 65 additions and 40 deletions.
diff --git a/.github/actions/setup-caches/action.yml b/.github/actions/setup-caches/action.yml
@@ -18,10 +18,12 @@ runs:
     # Setup CCache
     - name: Setup ccache
       uses: hendrikmuhs/[email protected]
+      with:
+        key: ${{ runner.os }}-${{ runner.arch }}
 
     - name: Setup ccache in shell
       shell: bash
-      run: echo 'PATH=/usr/lib/ccache:'"$PATH" >> $GITHUB_ENV
+      run: echo 'PATH=/usr/lib/ccache:/usr/local/opt/ccache/libexec:'"$PATH" >> $GITHUB_ENV
       if: ${{ runner.os != 'Windows' }}
       # TODO windows
 
@@ -31,10 +33,10 @@ runs:
       uses: actions/cache@v4
       with:
           path: |
-              ~/Library/Caches/Homebrew/boost--*
-              ~/Library/Caches/Homebrew/downloads/*--boost-*
-          key: brew-${{ hashFiles('CMakeLists.txt') }}
-          restore-keys: brew-
+              ~/Library/Caches/Homebrew/*
+              ~/Library/Caches/Homebrew/downloads/*
+          key: brew-${{ runner.os }}-${{ runner.arch }}-
+          restore-keys: brew-${{ runner.os }}-${{ runner.arch }}-
       if: ${{ runner.os == 'macOS' }}
 
     ################
@@ -48,12 +50,12 @@ runs:
         echo "VCPKG_DOWNLOADS=$HOME/vcpkg_download_cache" >> $GITHUB_ENV
       if: ${{ runner.os != 'Windows' }}
 
-    - name: Setup vcpkg cache (Linux)
+    - name: Setup vcpkg cache
       uses: actions/cache@v4
       with:
         path: |
           ~/vcpkg_cache
           ~/vcpkg_download_cache
-        key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json') }}
-        restore-keys: vcpkg-${{ runner.os }}
+        key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }}
+        restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}-
       if: ${{ runner.os != 'Windows' }}
diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml
@@ -21,14 +21,14 @@ runs:
     - name: Install python dependencies
       shell: bash
       run: make requirements
-      if: ${{ runner.os == 'Linux' }} # skip on linux, in docker
+      if: ${{ runner.os == 'Linux' }}
 
     ################
     # Linux # NOTE: skip for manylinux image
     # - name: Linux init steps
     #   shell: bash
     #   run: make dependencies-vcpkg
-    #   if: ${{ runner.os == 'Linux' }} # skip 
+    #   if: ${{ runner.os == 'Linux' }} # skip
 
     ################
     # Mac

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -4,9 +4,7 @@ on:
   push:
     branches:
       - main
-      - develop
-      - tkp/gh
-      - tkp/internal
+      - tkp/cachemac14
     tags:
       - v*
     paths-ignore:
@@ -52,7 +50,7 @@ jobs:
     # github.event.head_commit.message otherwise on pull
     # requests)
     initialize:
-      runs-on: ubuntu-20.04
+      runs-on: ubuntu-22.04
 
       outputs:
         COMMIT_MESSAGE: ${{ steps.setup.outputs.COMMIT_MSG }}
@@ -125,7 +123,7 @@ jobs:
       strategy:
         matrix:
           os:
-            - ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
+            - ubuntu-22.04
           python-version:
             - 3.9
 
@@ -167,11 +165,10 @@ jobs:
       strategy:
         matrix:
           os:
-            - ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
-            - macos-11 # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11-Readme.md
-            # - windows-2022 # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2022-Readme.md
-            # arm64 image
-            # - macos-13-xlarge # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md
+            - ubuntu-22.04 # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
+            - macos-12 # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
+            - macos-14 # https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md
+            # - windows-2022 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md
           python-version:
             - "3.8"
             - "3.9"
@@ -223,13 +220,22 @@ jobs:
               os: windows-2022
             # avoid unnecessary use of mac resources
             - is-full-run: false
-              os: macos-11
+              os: macos-12
               python-version: "3.8"
             - is-full-run: false
-              os: macos-11
+              os: macos-12
               python-version: "3.9"
             - is-full-run: false
-              os: macos-11
+              os: macos-12
+              python-version: "3.10"
+            - is-full-run: false
+              os: macos-14
+              python-version: "3.8"
+            - is-full-run: false
+              os: macos-14
+              python-version: "3.9"
+            - is-full-run: false
+              os: macos-14
               python-version: "3.10"
 
       runs-on: ${{ matrix.os }}
@@ -255,24 +261,37 @@ jobs:
           uses: ./.github/actions/setup-dependencies
           with:
             cibuildwheel: '${{ matrix.cibuildwheel }}'
+          if: ${{ runner.os == 'macOS' }}
 
         ########
         # Linux
         - name: Python Wheel Steps (Linux - cibuildwheel)
           run: make dist-py-cibw
           env:
             CIBW_BUILD: "${{ matrix.cibuildwheel }}-manylinux*"
-            CIBW_ENVIRONMENT_LINUX: CSP_MANYLINUX="ON" CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/host/home/runner/vcpkg_cache" VCPKG_DOWNLOADS="/host/home/runner/vcpkg_download_cache"
+            CIBW_ENVIRONMENT_LINUX: CSP_MANYLINUX="ON" CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="/host${{ env.VCPKG_DOWNLOADS }}"
+            CIBW_BUILD_VERBOSITY: 3
           if: ${{ runner.os == 'Linux' }}
 
         ########
         # Macos
-        - name: Python Build Steps (Macos)
+        - name: Python Build Steps (Macos x86)
           run: make dist-py-cibw
           env:
             CIBW_BUILD: "${{ matrix.cibuildwheel }}-macos*"
-            CIBW_ENVIRONMENT_MACOS: CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/Users/runner/vcpkg_cache" VCPKG_DOWNLOADS="/Users/runner/vcpkg_download_cache" CC="/usr/local/bin/gcc-13" CXX="/usr/local/bin/g++-13"
-          if: ${{ runner.os == 'macOS' }}
+            CIBW_ENVIRONMENT_MACOS: CC="/usr/local/bin/gcc-13" CXX="/usr/local/bin/g++-13" CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}"
+            CIBW_ARCHS_MACOS: x86_64
+            CIBW_BUILD_VERBOSITY: 3
+          if: ${{ matrix.os == 'macos-12' }}
+
+        - name: Python Build Steps (Macos arm)
+          run: make dist-py-cibw
+          env:
+            CIBW_BUILD: "${{ matrix.cibuildwheel }}-macos*"
+            CIBW_ENVIRONMENT_MACOS: PATH="/opt/homebrew/opt/bison/bin/:$PATH" CC="/opt/homebrew/bin/gcc-13" CXX="/opt/homebrew/bin/g++-13" LDFLAGS="-Wl,-ld_classic" CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}"
+            CIBW_ARCHS_MACOS: arm64
+            CIBW_BUILD_VERBOSITY: 3
+          if: ${{ matrix.os == 'macos-14' }}
 
         ##########
         # Windows
@@ -288,7 +307,7 @@ jobs:
         - name: Upload Wheel
           uses: actions/upload-artifact@v4
           with:
-            name: csp-dist-${{ runner.os }}-${{ matrix.python-version }}
+            name: csp-dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}
             path: wheelhouse/*.whl
 
     #############################
@@ -310,7 +329,7 @@ jobs:
       strategy:
         matrix:
           os:
-            - ubuntu-20.04
+            - ubuntu-22.04
           python-version:
             - 3.9
 
@@ -361,10 +380,10 @@ jobs:
       strategy:
         matrix:
           os:
-            - ubuntu-20.04
-            - macos-11
+            - ubuntu-22.04
+            - macos-12
+            - macos-14
             # - windows-2022
-            # - macos-13-xlarge # arm64 image
           python-version:
             - 3.8
             - 3.9
@@ -382,7 +401,10 @@ jobs:
 
             # Exclude macOS builds for now
             - is-full-run: false
-              os: macos-11
+              os: macos-12
+
+            - is-full-run: false
+              os: macos-14
 
             # Exclude Python 3.8, 3.10, 3.11 builds
             - is-full-run: false
@@ -425,7 +447,7 @@ jobs:
         - name: Download wheel
           uses: actions/download-artifact@v4
           with:
-            name: csp-dist-${{ runner.os }}-${{ matrix.python-version }}
+            name: csp-dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}
 
         - name: Install wheel (Linux)
           run: python -m pip install -U *manylinux2014*.whl --target .
@@ -469,7 +491,7 @@ jobs:
     #   strategy:
     #     matrix:
     #       os:
-    #         - ubuntu-20.04
+    #         - ubuntu-22.04
     #       python-version:
     #         - 3.9
 

diff --git a/Makefile b/Makefile
@@ -154,11 +154,11 @@ clean: ## clean the repository
 .PHONY: dependencies-mac dependencies-debian dependencies-fedora dependencies-vcpkg dependencies-win
 
 dependencies-mac:  ## install dependencies for mac
-	HOMEBREW_NO_AUTO_UPDATE=1 brew install bison cmake flex # gcc@12
+	HOMEBREW_NO_AUTO_UPDATE=1 brew install bison cmake flex make ninja # gcc@13
+	brew unlink bison flex && brew link --force bison flex
 
 dependencies-debian:  ## install dependencies for linux
 	apt-get install -y automake bison cmake curl flex ninja-build tar unzip zip
-	# libabsl-dev libarrow-dev libparquet-dev libthrift-dev
 
 dependencies-fedora:  ## install dependencies for linux
 	yum install -y automake bison cmake curl flex perl-IPC-Cmd tar unzip zip

diff --git a/pyproject.toml b/pyproject.toml
@@ -77,15 +77,16 @@ test-requires = [
 ]
 
 [tool.cibuildwheel.linux]
-before-all ="make dependencies-fedora && make dependencies-vcpkg"
+before-all ="make dependencies-fedora"
 environment = {CSP_MANYLINUX="ON"}
 repair-wheel-command = "auditwheel -v show {wheel} && LD_LIBRARY_PATH=/project/csp/lib auditwheel -v repair -w {dest_dir} {wheel}"
 skip = "*i686 musllinux*"
 
 [tool.cibuildwheel.macos]
-before-all ="make dependencies-mac && make dependencies-vcpkg"
+before-all ="echo $PATH && make dependencies-mac"
 archs = "x86_64"  # NOTE: we use gcc and we cannot cross compile for now
-environment = {CFLAGS="-I/usr/local/include -L/usr/local/lib", CXXFLAGS="-I/usr/local/include -L/usr/local/lib", LDFLAGS="-L/usr/local/lib"}
+# environment = {CFLAGS="-I/usr/local/include -L/usr/local/lib", CXXFLAGS="-I/usr/local/include -L/usr/local/lib", LDFLAGS="-L/usr/local/lib"}
+environment = {}
 
 [tool.cibuildwheel.windows]
 archs = "AMD64"