Use caching to speed up testdata fetching, fix upstream builds, scipy suggestions

.github/workflows/main.yml

@@ -40,8 +40,7 @@ jobs:
       (github.event_name == 'push')
     strategy:
       matrix:
-        python-version:
-          - "3.9"
+        python-version: [ "3.9" ]
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
@@ -58,9 +57,17 @@ jobs:
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
       - name: Install CI libraries
         run: |
           python -m pip install --require-hashes -r CI/requirements_ci.txt
+      - name: Environment Caching
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+            .tox
+          key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-lint
       - name: Run pylint
         run: |
           python -m pylint --rcfile=.pylintrc.toml --disable=import-error --exit-zero xclim
@@ -69,13 +76,14 @@ jobs:
           python -m tox -e lint
 
   test-preliminary:
-    name: Python${{ matrix.python-version }} (ubuntu-latest)
+    name: Python${{ matrix.python-version }} (${{ matrix.os }})
     needs: lint
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version:
-          - "3.9"
+        os: [ 'ubuntu-latest' ]
+        python-version: [ "3.9" ]
+        testdata-cache: [ '~/.cache/xclim-testdata' ]
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
@@ -95,9 +103,17 @@ jobs:
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
       - name: Install CI libraries
         run: |
           python -m pip install --require-hashes -r CI/requirements_ci.txt
+      - name: Environment Caching
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+            .tox
+          key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }}
       - name: Test with tox
         run: |
           python -m tox -- -m 'not slow'
@@ -118,39 +134,47 @@ jobs:
       matrix:
         include:
           # Linux builds
-          - os: ubuntu-latest
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
             markers: -m 'not slow'
             python-version: "3.10"
             tox-env: standard
-          - os: ubuntu-latest
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
             markers: -m 'not slow'
             python-version: "3.11"
             tox-env: standard
-          - os: ubuntu-latest
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
             markers: -m 'not slow'
             python-version: "3.12"
             tox-env: standard
           # Windows builds
-          - os: windows-latest
+          - os: 'windows-latest'
+            testdata-cache: 'C:\Users\runneradmin\AppData\Local\xclim-testdata\xclim-testdata\Cache'
             markers: -m 'not slow'
             python-version: "3.9"
             tox-env: py39-coverage-prefetch  # Test data prefetch is needed for Windows
           # macOS builds
-          - os: macos-latest
-            python-version: "3.10"
+          - os: 'macos-latest'
+            testdata-cache: '~/Library/Caches/xclim-testdata'
             markers: ''  # Slow tests
+            python-version: "3.10"
             tox-env: py310-coverage-extras
           # Specialized tests
-          - os: ubuntu-latest
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
             markers: -m 'not requires_internet and not slow'
             python-version: "3.9"
             tox-env: py39-coverage-offline-prefetch
-          - os: ubuntu-latest
-            markers: ''
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
+            markers: '' # No markers for notebooks
             python-version: "3.10"
             tox-env: notebooks
-          - os: ubuntu-latest
-            markers: ''
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
+            markers: '' # No markers for doctests
             python-version: "3.12"
             tox-env: doctests
     steps:
@@ -182,9 +206,19 @@ jobs:
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
       - name: Install CI libraries
         run: |
           python -m pip install --require-hashes -r CI/requirements_ci.txt
+      - name: Environment Caching
+        # if prefetch is not in tox-env
+        if: contains(matrix.tox-env, 'prefetch') == false
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+            .tox
+          key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }}
       - name: Test with tox
         if: ${{ matrix.tox-env == 'standard' }}
         run: |
@@ -212,8 +246,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ 'ubuntu-latest' ]
         python-version: [ "3.9", "3.12" ]
+        testdata-cache: [ '~/.cache/xclim-testdata' ]
     defaults:
       run:
         shell: bash -l {0}
@@ -247,6 +282,12 @@ jobs:
       - name: Micromamba version
         run: |
           echo "micromamba: $(micromamba --version)"
+      - name: Test Data Caching
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+          key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-conda-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }}
       - name: Install xclim
         run: |
           python -m pip install --no-user --editable .
@@ -284,18 +325,13 @@ jobs:
       - test-pypi
       - test-conda
     runs-on: ubuntu-latest
-    container: python:3-slim
     steps:
-      - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: Harden Runner
+        uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1
         with:
-          sparse-checkout: |
-            CI/requirements_ci.txt
-      - name: Install CI libraries
-        run: |
-          python -m pip install --require-hashes -r CI/requirements_ci.txt
-      - name: Coveralls finished
-        run: |
-          python -m coveralls --finish
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          disable-sudo: true
+          egress-policy: audit
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0
+        with:
+          parallel-finished: true

.github/workflows/upstream.yml

@@ -32,8 +32,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-          - "3.10"
+        python-version: [ "3.12" ]
+        testdata-cache: [ '~/.cache/xclim-testdata' ]
     defaults:
       run:
         shell: bash -l {0}
@@ -66,8 +66,8 @@ jobs:
           create-args: >-
             eigen
             pybind11
-            python=${{ matrix.python-version }}
             pytest-reportlog
+            python=${{ matrix.python-version }}
       - name: Micromamba version
         run: |
           echo "micromamba: $(micromamba --version)"
@@ -84,6 +84,12 @@ jobs:
           micromamba list
           xclim show_version_info
           python -m pip check || true
+      - name: Test Data Caching
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+          key: ${{ runner.os }}-xclim-testdata-upstream-${{ hashFiles('pyproject.toml', 'tox.ini') }}
       - name: Run Tests
         if: success()
         id: status
@@ -97,4 +103,5 @@ jobs:
           && github.repository_owner == 'Ouranosinc'
         uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1.3.0
         with:
+          issue-title: "⚠️ Nightly upstream-dev CI failed for Python${{ matrix.python-version }} ⚠️"
           log-path: output-${{ matrix.python-version }}-log.jsonl

CHANGELOG.rst

@@ -43,6 +43,10 @@ Internal changes
 * ``xclim.core`` now uses absolute imports for clarity and some objects commonly used in the module have been moved to hidden submodules. (:issue:`1719`, :pull:`1881`).
 * ``xclim.core.indicator.Parameter`` has a new attribute ``compute_name`` while ``xclim.core.indicator.Indicator`` lost its ``_variable_mapping``. The translation from parameter (and variable) names in the indicator to the names on the compute function is handled by ``Indicator._get_compute_args``. (:pull:`1885`).
 
+CI changes
+^^^^^^^^^^
+* The `pip` cache, `tox` environments, and the `xclim-testdata` cache are now saved between workflow runs (using `actions/cache`) to reduce the time spent installing dependencies and downloading testing data. (:pull:`1906`).
+
 v0.52.0 (2024-08-08)
 --------------------
 Contributors to this version: David Huard (:user:`huard`), Trevor James Smith (:user:`Zeitsperre`), Hui-Min Wang (:user:`Hem-W`), Éric Dupuis (:user:`coxipi`), Sarah Gammon (:user:`SarahG-579462`), Pascal Bourgault (:user:`aulemahal`), Juliette Lavoie (:user:`juliettelavoie`), Adrien Lamarche (:user:`LamAdr`).

CI/requirements_upstream.txt

@@ -1,6 +1,5 @@
 bottleneck @ git+https://github.com/pydata/bottleneck.git@master
 cftime @ git+https://github.com/Unidata/cftime.git@master
 flox @ git+https://github.com/xarray-contrib/flox.git@main
-numba @ git+https://github.com/numba/numba.git@main
 numpy @ git+https://github.com/numpy/numpy.git@main
 xarray @ git+https://github.com/pydata/xarray.git@main