From 5da4181749b361dcabf7d6d815e31a710fb0a6a8 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:26:15 -0400 Subject: [PATCH 01/25] use caching to speed up testdata fetching --- .github/workflows/main.yml | 48 ++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 464862983..a968381a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,6 +58,7 @@ jobs: uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt @@ -95,6 +96,7 @@ jobs: uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt @@ -118,39 +120,47 @@ jobs: matrix: include: # Linux builds - - os: ubuntu-latest + - os: 'ubuntu-latest' + testdata-cache: '~/.cache/xclim-testdata' markers: -m 'not slow' python-version: "3.10" tox-env: standard - - os: ubuntu-latest + - os: 'ubuntu-latest' + testdata-cache: '~/.cache/xclim-testdata' markers: -m 'not slow' python-version: "3.11" tox-env: standard - - os: ubuntu-latest + - os: 'ubuntu-latest' + testdata-cache: '~/.cache/xclim-testdata' markers: -m 'not slow' python-version: "3.12" tox-env: standard # Windows builds - - os: windows-latest + - os: 'windows-latest' + testdata-cache: 'C:\Users\runneradmin\AppData\Local\xclim-testdata\xclim-testdata\Cache' markers: -m 'not slow' python-version: "3.9" tox-env: py39-coverage-prefetch # Test data prefetch is needed for Windows # macOS builds - - os: macos-latest - python-version: "3.10" + - os: 'macos-latest' + testdata-cache: '~/Library/Caches/xclim-testdata' markers: '' # Slow tests + python-version: "3.10" tox-env: py310-coverage-extras # Specialized tests - - os: ubuntu-latest + - os: 'ubuntu-latest' + testdata-cache: '~/.cache/xclim-testdata' markers: -m 'not requires_internet and not slow' python-version: "3.9" tox-env: py39-coverage-offline-prefetch - - os: ubuntu-latest - markers: '' + - os: 'ubuntu-latest' + testdata-cache: '~/.cache/xclim-testdata' + markers: '' # No markers for notebooks python-version: "3.10" tox-env: notebooks - - os: ubuntu-latest - markers: '' + - os: 'ubuntu-latest' + testdata-cache: '~/.cache/xclim-testdata' + markers: '' # No markers for doctests python-version: "3.12" tox-env: doctests steps: @@ -182,9 +192,17 @@ jobs: uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Test Data Cache + # if prefetch is not in tox-env + if: ${{ matrix.tox-env != *prefetch* }} + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ matrix.testdata-cache }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }} - name: Test with tox if: ${{ matrix.tox-env == 'standard' }} run: | @@ -212,8 +230,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-latest ] + os: [ 'ubuntu-latest' ] python-version: [ "3.9", "3.12" ] + testdata-cache: [ '~/.cache/xclim-testdata' ] defaults: run: shell: bash -l {0} @@ -247,6 +266,11 @@ jobs: - name: Micromamba version run: | echo "micromamba: $(micromamba --version)" + - name: Test Data Cache + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ matrix.testdata-cache }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }} - name: Install xclim run: | python -m pip install --no-user --editable . From e6f6977813a28ef3f987618d58add58f9a3d70ab Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:31:42 -0400 Subject: [PATCH 02/25] fix expression --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a968381a9..d52126b9b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -198,7 +198,7 @@ jobs: python -m pip install --require-hashes -r CI/requirements_ci.txt - name: Test Data Cache # if prefetch is not in tox-env - if: ${{ matrix.tox-env != *prefetch* }} + if: contains(matrix.tox-env, 'prefetch') == false uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: path: ${{ matrix.testdata-cache }} From 539b8b8040ea61994eb4ad6ffe9335e4da6248f3 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:36:48 -0400 Subject: [PATCH 03/25] add caching to preliminary build --- .github/workflows/main.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d52126b9b..a504e3edb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -75,8 +75,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: - - "3.9" + os: [ 'ubuntu-latest' ] + python-version: [ "3.9" ] + testdata-cache: [ '~/.cache/xclim-testdata' ] steps: - name: Harden Runner uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 @@ -100,6 +101,11 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Test Data Cache + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ matrix.testdata-cache }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }} - name: Test with tox run: | python -m tox -- -m 'not slow' From 18ec3feefdbab0d714081ac895db2cd5b16e22a2 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 5 Sep 2024 15:43:07 -0400 Subject: [PATCH 04/25] update CHANGELOG.rst and more caching --- .github/workflows/main.yml | 29 ++++++++++++++++++++--------- CHANGELOG.rst | 4 ++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a504e3edb..73c076db2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,8 +40,8 @@ jobs: (github.event_name == 'push') strategy: matrix: - python-version: - - "3.9" + python-version: [ "3.9" ] + tox-env: [ 'lint' ] steps: - name: Harden Runner uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 @@ -62,15 +62,22 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Test Data Cache + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: | + ${{ matrix.testdata-cache }} + .tox/${{ matrix.tox-env }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ matrix.tox-env }} - name: Run pylint run: | python -m pylint --rcfile=.pylintrc.toml --disable=import-error --exit-zero xclim - name: Run linting suite run: | - python -m tox -e lint + python -m tox -e ${{ matrix.tox-env }} test-preliminary: - name: Python${{ matrix.python-version }} (ubuntu-latest) + name: Python${{ matrix.python-version }} (${{ matrix.os }}) needs: lint runs-on: ubuntu-latest strategy: @@ -104,8 +111,10 @@ jobs: - name: Test Data Cache uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: - path: ${{ matrix.testdata-cache }} - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }} + path: | + ${{ matrix.testdata-cache }} + .tox + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-preliminary-${{ matrix.python-version }} - name: Test with tox run: | python -m tox -- -m 'not slow' @@ -207,8 +216,10 @@ jobs: if: contains(matrix.tox-env, 'prefetch') == false uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: - path: ${{ matrix.testdata-cache }} - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }} + path: | + ${{ matrix.testdata-cache }} + .tox + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ matrix.tox-env }}-${{ matrix.python-version }} - name: Test with tox if: ${{ matrix.tox-env == 'standard' }} run: | @@ -276,7 +287,7 @@ jobs: uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: path: ${{ matrix.testdata-cache }} - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-conda-${{ matrix.python-version }} - name: Install xclim run: | python -m pip install --no-user --editable . diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 281d721e1..92e7d25dd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -33,6 +33,10 @@ Internal changes * The codebase has been adjusted to address many `pylint`-related warnings and errors. In some cases, `casting` was used to redefine some `numpy` and `xarray` objects. (:issue:`1719`, :pull:`1881`). * ``xclim.core`` now uses absolute imports for clarity and some objects commonly used in the module have been moved to hidden submodules. (:issue:`1719`, :pull:`1881`). +CI changes +^^^^^^^^^^ +* The `pip` cache and the `xclim-testdata` are now cached between workflow runs (using `actions/cache`) to reduce the time spent installing dependencies and downloading testing data. (:pull:`1889`). + v0.52.0 (2024-08-08) -------------------- Contributors to this version: David Huard (:user:`huard`), Trevor James Smith (:user:`Zeitsperre`), Hui-Min Wang (:user:`Hem-W`), Éric Dupuis (:user:`coxipi`), Sarah Gammon (:user:`SarahG-579462`), Pascal Bourgault (:user:`aulemahal`), Juliette Lavoie (:user:`juliettelavoie`), Adrien Lamarche (:user:`LamAdr`). From a95ab63a87ffe98206c9b7431a5fca990c59814d Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 5 Sep 2024 15:55:24 -0400 Subject: [PATCH 05/25] adjustments --- .github/workflows/main.yml | 7 +++---- CHANGELOG.rst | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 73c076db2..95330fd57 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,7 +41,6 @@ jobs: strategy: matrix: python-version: [ "3.9" ] - tox-env: [ 'lint' ] steps: - name: Harden Runner uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 @@ -67,14 +66,14 @@ jobs: with: path: | ${{ matrix.testdata-cache }} - .tox/${{ matrix.tox-env }} - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ matrix.tox-env }} + .tox + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-lint - name: Run pylint run: | python -m pylint --rcfile=.pylintrc.toml --disable=import-error --exit-zero xclim - name: Run linting suite run: | - python -m tox -e ${{ matrix.tox-env }} + python -m tox -e lint test-preliminary: name: Python${{ matrix.python-version }} (${{ matrix.os }}) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 92e7d25dd..927a0e821 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,7 +35,7 @@ Internal changes CI changes ^^^^^^^^^^ -* The `pip` cache and the `xclim-testdata` are now cached between workflow runs (using `actions/cache`) to reduce the time spent installing dependencies and downloading testing data. (:pull:`1889`). +* The `pip` cache, `tox` environments, and the `xclim-testdata` cache are now saved between workflow runs (using `actions/cache`) to reduce the time spent installing dependencies and downloading testing data. (:pull:`1906`). v0.52.0 (2024-08-08) -------------------- From 8a01a043968503b6359fd7b36876376fbce58e07 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:08:17 -0400 Subject: [PATCH 06/25] more streamlined caching --- .github/workflows/main.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95330fd57..59a4f206b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,7 +67,7 @@ jobs: path: | ${{ matrix.testdata-cache }} .tox - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-lint + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }} - name: Run pylint run: | python -m pylint --rcfile=.pylintrc.toml --disable=import-error --exit-zero xclim @@ -113,7 +113,7 @@ jobs: path: | ${{ matrix.testdata-cache }} .tox - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-preliminary-${{ matrix.python-version }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }} - name: Test with tox run: | python -m tox -- -m 'not slow' @@ -218,7 +218,7 @@ jobs: path: | ${{ matrix.testdata-cache }} .tox - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ matrix.tox-env }}-${{ matrix.python-version }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }} - name: Test with tox if: ${{ matrix.tox-env == 'standard' }} run: | @@ -285,8 +285,9 @@ jobs: - name: Test Data Cache uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: - path: ${{ matrix.testdata-cache }} - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-conda-${{ matrix.python-version }} + path: | + ${{ matrix.testdata-cache }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-conda - name: Install xclim run: | python -m pip install --no-user --editable . From 51d377a0263ccc6663dff47f09b2fc54bf48f9e3 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:25:43 -0400 Subject: [PATCH 07/25] better naming --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 59a4f206b..91f9bbc2e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,7 +61,7 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt - - name: Test Data Cache + - name: Environment Caching uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: path: | @@ -107,7 +107,7 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt - - name: Test Data Cache + - name: Environment Caching uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: path: | @@ -210,7 +210,7 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt - - name: Test Data Cache + - name: Environment Caching # if prefetch is not in tox-env if: contains(matrix.tox-env, 'prefetch') == false uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 @@ -282,7 +282,7 @@ jobs: - name: Micromamba version run: | echo "micromamba: $(micromamba --version)" - - name: Test Data Cache + - name: Test Data Caching uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: path: | From 805b43e0a90dd889d136743bd8ead706904f421a Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 5 Sep 2024 17:31:23 -0400 Subject: [PATCH 08/25] final adjustments --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91f9bbc2e..d42c4d98c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,7 +67,7 @@ jobs: path: | ${{ matrix.testdata-cache }} .tox - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-lint - name: Run pylint run: | python -m pylint --rcfile=.pylintrc.toml --disable=import-error --exit-zero xclim @@ -113,7 +113,7 @@ jobs: path: | ${{ matrix.testdata-cache }} .tox - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }} - name: Test with tox run: | python -m tox -- -m 'not slow' @@ -218,7 +218,7 @@ jobs: path: | ${{ matrix.testdata-cache }} .tox - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }} + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }} - name: Test with tox if: ${{ matrix.tox-env == 'standard' }} run: | @@ -287,7 +287,7 @@ jobs: with: path: | ${{ matrix.testdata-cache }} - key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-conda + key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-conda-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }} - name: Install xclim run: | python -m pip install --no-user --editable . From 4b6fb1ff10f6b4eabdcf8c7ccb0ed9c0494d54fd Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 6 Sep 2024 10:29:46 -0400 Subject: [PATCH 09/25] use coveralls app --- .github/workflows/main.yml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d42c4d98c..01a5d2ef9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -325,18 +325,13 @@ jobs: - test-pypi - test-conda runs-on: ubuntu-latest - container: python:3-slim steps: - - name: Checkout Repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Harden Runner + uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1 with: - sparse-checkout: | - CI/requirements_ci.txt - - name: Install CI libraries - run: | - python -m pip install --require-hashes -r CI/requirements_ci.txt - - name: Coveralls finished - run: | - python -m coveralls --finish - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + disable-sudo: true + egress-policy: audit + - name: Coveralls Finished + uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0 + with: + parallel-finished: true From 61686dfc23222b6ac330bf93d75acd7810d75fff Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 6 Sep 2024 15:02:36 -0400 Subject: [PATCH 10/25] use caching in upstream.yml --- .github/workflows/upstream.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 9f91caf47..13ac74d75 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -32,8 +32,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: - - "3.10" + python-version: [ "3.9", "3.12" ] + testdata-cache: [ '~/.cache/xclim-testdata' ] defaults: run: shell: bash -l {0} @@ -66,8 +66,8 @@ jobs: create-args: >- eigen pybind11 - python=${{ matrix.python-version }} pytest-reportlog + python=${{ matrix.python-version }} - name: Micromamba version run: | echo "micromamba: $(micromamba --version)" @@ -84,6 +84,12 @@ jobs: micromamba list xclim show_version_info python -m pip check || true + - name: Test Data Caching + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: | + ${{ matrix.testdata-cache }} + key: ${{ runner.os }}-xclim-testdata-upstream-${{ hashFiles('pyproject.toml', 'tox.ini') }} - name: Run Tests if: success() id: status @@ -97,4 +103,5 @@ jobs: && github.repository_owner == 'Ouranosinc' uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1.3.0 with: + issue-title: "⚠️ Nightly upstream-dev CI failed for Python${{ matrix.python-version }} ⚠️" log-path: output-${{ matrix.python-version }}-log.jsonl From b087effd5e7b5344bfd999778161796f28628d3f Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 6 Sep 2024 16:20:12 -0400 Subject: [PATCH 11/25] fix silently failing upstream dependency installation step --- .github/workflows/upstream.yml | 2 +- CI/requirements_upstream.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 13ac74d75..499118151 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.12" ] + python-version: [ "3.12" ] testdata-cache: [ '~/.cache/xclim-testdata' ] defaults: run: diff --git a/CI/requirements_upstream.txt b/CI/requirements_upstream.txt index c5ebda67e..9650a59df 100644 --- a/CI/requirements_upstream.txt +++ b/CI/requirements_upstream.txt @@ -1,6 +1,5 @@ bottleneck @ git+https://github.com/pydata/bottleneck.git@master cftime @ git+https://github.com/Unidata/cftime.git@master flox @ git+https://github.com/xarray-contrib/flox.git@main -numba @ git+https://github.com/numba/numba.git@main numpy @ git+https://github.com/numpy/numpy.git@main xarray @ git+https://github.com/pydata/xarray.git@main From e9adfe44f26965564e9801a40ac575e65413857b Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:25:58 -0400 Subject: [PATCH 12/25] replace isort with `ruff`, add pygrep-hooks and mdformat, update ruff configuration --- Makefile | 1 - pyproject.toml | 11 ++++------- tox.ini | 2 -- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 3e51fad2d..898f27d5a 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,6 @@ clean-test: ## remove test and coverage artifacts lint: ## check style with flake8 and black black --check xclim tests - isort --check xclim tests ruff check xclim tests flake8 --config=.flake8 xclim tests vulture xclim tests diff --git a/pyproject.toml b/pyproject.toml index d363a3385..1a5cfd9f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,12 +223,6 @@ exclude = [ "pylintrc" ] -[tool.isort] -profile = "black" -py_version = 39 -append_only = true -add_imports = "from __future__ import annotations" - [tool.mypy] python_version = 3.9 show_error_codes = true @@ -275,7 +269,6 @@ markers = [ [tool.ruff] src = ["xclim"] line-length = 150 -target-version = "py39" exclude = [ ".git", "build", @@ -296,18 +289,22 @@ extend-select = [ "RUF022" # unsorted-dunder-all ] ignore = [ + "B028", # no-explicit-stacklevel "D205", # blank-line-after-summary "D400", # ends-in-period "D401" # non-imperative-mood ] preview = true select = [ + "B", # bugbear "C90", # mccabe-complexity "D", # docstrings "E", # pycodestyle errors "F", # pyflakes + "I", # imports "N802", # invalid-function-name "S", # bandit + "UP", # pyupgrade "W" # pycodestyle warnings ] diff --git a/tox.ini b/tox.ini index d88f67ca9..d59e867cd 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,6 @@ deps = flake8-rst-docstrings black[jupyter]==24.4.2 blackdoc==0.3.9 - isort==5.13.2 nbqa ruff==0.4.10 vulture==2.11 @@ -44,7 +43,6 @@ deps = commands_pre = commands = black --check xclim tests - isort --check xclim tests ruff check xclim tests flake8 --config=.flake8 xclim tests vulture xclim tests From ff8f98063d6ce72815ec7660c468dfe680fac291 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:26:37 -0400 Subject: [PATCH 13/25] update pre-commit config --- .pre-commit-config.yaml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56c460a73..d8a5a81f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,15 +36,11 @@ repos: rev: 24.8.0 hooks: - id: black - - repo: https://github.com/PyCQA/isort - rev: 5.13.2 - hooks: - - id: isort - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.6.3 hooks: - id: ruff - args: [ '--fix' ] + args: [ '--fix', '--show-fixes' ] - repo: https://github.com/pylint-dev/pylint rev: v3.2.7 hooks: @@ -68,20 +64,34 @@ repos: additional_dependencies: [ 'pyupgrade==3.16.0' ] - id: nbqa-black additional_dependencies: [ 'black==24.4.2' ] - - id: nbqa-isort - additional_dependencies: [ 'isort==5.13.2' ] - repo: https://github.com/kynan/nbstripout rev: 0.7.1 hooks: - id: nbstripout files: '.ipynb' args: [ '--extra-keys', 'metadata.kernelspec' ] + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: +# - id: python-check-blanket-noqa +# - id: python-check-blanket-type-ignore + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations + - id: rst-directive-colons + - id: rst-inline-touching-normal + - id: text-unicode-replacement-char + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.17 + hooks: + - id: mdformat - repo: https://github.com/keewis/blackdoc rev: v0.3.9 hooks: - id: blackdoc - additional_dependencies: [ 'black==24.4.2' ] + additional_dependencies: [ 'black==24.8.0' ] exclude: '(xclim/indices/__init__.py|docs/installation.rst)' + - id: blackdoc-autoupdate-black - repo: https://github.com/codespell-project/codespell rev: v2.3.0 hooks: From 8623cd67cf46cf4a6e6556b725bb156d80207dac Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:28:51 -0400 Subject: [PATCH 14/25] update markdown files --- .github/ISSUE_TEMPLATE.md | 12 ++++++++---- CODE_OF_CONDUCT.md | 24 ++++++++++++------------ SECURITY.md | 36 +++++++++++++++++++----------------- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 8e0001803..222c2696d 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,21 +1,25 @@ -* xclim version: -* Python version: -* Operating System: +- xclim version: +- Python version: +- Operating System: ### Description + - ### What I Did + + ``` $ pip install foo --bar ``` ### What I Received + + ``` Traceback (most recent call last): File "/path/to/file/script.py", line 3326, in run_code diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 5233d8519..e0ee82bd0 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -17,23 +17,23 @@ diverse, inclusive, and healthy community. Examples of behavior that contributes to a positive environment for our community include: -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience -* Focusing on what is best not just for us as individuals, but for the +- Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: -* The use of sexualized language or imagery, and sexual attention or +- The use of sexualized language or imagery, and sexual attention or advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a +- Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities @@ -121,8 +121,8 @@ https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). -[homepage]: https://www.contributor-covenant.org - For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. + +[homepage]: https://www.contributor-covenant.org diff --git a/SECURITY.md b/SECURITY.md index 6f1c898b2..85da0b5bb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,27 +12,29 @@ Please follow these steps to report a security vulnerability: 1. **Email**: Email [github-support@ouranos.ca](mailto:github-support@ouranos.ca) with a detailed description of the vulnerability. If applicable, please include any steps or a proof-of-concept to help us understand and reproduce the issue. -2. **Encryption (Optional)**: If you are concerned about the sensitivity of the information you are sharing, you can use the PGP key found below to encrypt your communication. +1. **Encryption (Optional)**: If you are concerned about the sensitivity of the information you are sharing, you can use the PGP key found below to encrypt your communication. -3. **Response**: We will acknowledge your email within 48 hours and work with you to understand and confirm the vulnerability. +1. **Response**: We will acknowledge your email within 48 hours and work with you to understand and confirm the vulnerability. -4. **Fix and Disclosure**: Once the vulnerability is confirmed, we will work to address it promptly. We appreciate your patience as we investigate and implement a fix. Once resolved, we will coordinate the disclosure and provide credit to the reporter unless they prefer to remain anonymous. +1. **Fix and Disclosure**: Once the vulnerability is confirmed, we will work to address it promptly. We appreciate your patience as we investigate and implement a fix. Once resolved, we will coordinate the disclosure and provide credit to the reporter unless they prefer to remain anonymous. ## PGP Encryption Key You can use the following PGP key to encrypt your communications with us: - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mDMEZamQrhYJKwYBBAHaRw8BAQdA+saPvmvr1MYe1nQy3n3QDcRE9T7UzTJ1XH31 - EI4Zb6u0Mk91cmFub3MgR2l0SHViIFN1cHBvcnQgPGdpdGh1Yi1zdXBwb3J0QG91 - cmFub3MuY2E+iJkEExYKAEEWIQSeAu+Cbjupx79jy9VeVFD6o5TVcwUCZamQrgIb - AwUJCWYBgAULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAKCRBeVFD6o5TVc4ho - AQDXjDkx0b3A7yl6PQ4hBJ2uYzw0UWbml7mUwVdhMmdZkQD/VJZQNWrCQeOtYEM8 - icZJYwR/OsKFOWqlDytusGGtjwa4OARlqZCuEgorBgEEAZdVAQUBAQdAa41Zabjz - P9O+p6tI69Cnft6U5om3+qCcMo8amTqauH0DAQgHiH4EGBYKACYWIQSeAu+Cbjup - x79jy9VeVFD6o5TVcwUCZamQrgIbDAUJCWYBgAAKCRBeVFD6o5TVcwmaAQClDxW6 - 2gir7lhRXAcO+vmRImpGd29TrkcQVh+ak7VlwQEA706d7Kusiorlf/h8pLSoNMmS - kuLGmHpUJ8NVGppU+wo= - =wuxr - -----END PGP PUBLIC KEY BLOCK----- +``` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEZamQrhYJKwYBBAHaRw8BAQdA+saPvmvr1MYe1nQy3n3QDcRE9T7UzTJ1XH31 +EI4Zb6u0Mk91cmFub3MgR2l0SHViIFN1cHBvcnQgPGdpdGh1Yi1zdXBwb3J0QG91 +cmFub3MuY2E+iJkEExYKAEEWIQSeAu+Cbjupx79jy9VeVFD6o5TVcwUCZamQrgIb +AwUJCWYBgAULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAKCRBeVFD6o5TVc4ho +AQDXjDkx0b3A7yl6PQ4hBJ2uYzw0UWbml7mUwVdhMmdZkQD/VJZQNWrCQeOtYEM8 +icZJYwR/OsKFOWqlDytusGGtjwa4OARlqZCuEgorBgEEAZdVAQUBAQdAa41Zabjz +P9O+p6tI69Cnft6U5om3+qCcMo8amTqauH0DAQgHiH4EGBYKACYWIQSeAu+Cbjup +x79jy9VeVFD6o5TVcwUCZamQrgIbDAUJCWYBgAAKCRBeVFD6o5TVcwmaAQClDxW6 +2gir7lhRXAcO+vmRImpGd29TrkcQVh+ak7VlwQEA706d7Kusiorlf/h8pLSoNMmS +kuLGmHpUJ8NVGppU+wo= +=wuxr +-----END PGP PUBLIC KEY BLOCK----- +``` From 6d45285078d5b04558528ba4ac77b1697a72c123 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:48:46 -0400 Subject: [PATCH 15/25] fix markdown formatting rules --- ...blish-mastodon.template.md => publish-mastodon-template.md} | 1 - .pre-commit-config.yaml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/{publish-mastodon.template.md => publish-mastodon-template.md} (99%) diff --git a/.github/publish-mastodon.template.md b/.github/publish-mastodon-template.md similarity index 99% rename from .github/publish-mastodon.template.md rename to .github/publish-mastodon-template.md index 308374f11..9f04b6e49 100644 --- a/.github/publish-mastodon.template.md +++ b/.github/publish-mastodon-template.md @@ -2,4 +2,3 @@ New #xclim release: {{ .version }} 🎉 Latest source code available at: https://github.com/Ouranosinc/xclim/releases/tag/{{ .version }} Check out the docs for more information: https://xclim.readthedocs.io/en/stable/ - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8a5a81f5..48b1a38c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer - exclude: '.ipynb|.github/publish-mastodon.template.md' + exclude: '.ipynb|.github/publish-mastodon-template.md' - id: check-json - id: check-toml - id: check-yaml @@ -85,6 +85,7 @@ repos: rev: 0.7.17 hooks: - id: mdformat + exclude: '.github/\w+.md|docs/paper/paper.md' - repo: https://github.com/keewis/blackdoc rev: v0.3.9 hooks: From 30e73806f9e364b33bace7683d842700aa0d4c22 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:49:56 -0400 Subject: [PATCH 16/25] fix remaining non-code files --- .github/workflows/publish-mastodon.yml | 2 +- CHANGELOG.rst | 4 ++-- pyproject.toml | 15 +++++++++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish-mastodon.yml b/.github/workflows/publish-mastodon.yml index 4ba46f5d9..36373f557 100644 --- a/.github/workflows/publish-mastodon.yml +++ b/.github/workflows/publish-mastodon.yml @@ -70,7 +70,7 @@ jobs: id: render_template uses: chuhlomin/render-template@807354a04d9300c9c2ac177c0aa41556c92b3f75 # v1.10 with: - template: .github/publish-mastodon.template.md + template: .github/publish-mastodon-template.md vars: | version: ${{ env.version }} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 927a0e821..26a81f8f6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -355,7 +355,7 @@ New features and enhancements * Optimized and noticeably faster calculation. * Can be computed in two steps: first compute fit parameters with ``xclim.indices.stats.standardized_index_fit_params``, then use the output in the standardized indices functions. * The standardized index values are now clipped to ±8.21. This reflects the ``float64`` precision of the computation when cumulative distributed function values are inverted to a normal distribution and avoids returning infinite values. - * An offset parameter is now available to account for negative water balance values``xclim.indices.standardized_precipitation_evapotranspiration_index``. + * An offset parameter is now available to account for negative water balance values ``xclim.indices.standardized_precipitation_evapotranspiration_index``. Bug fixes ^^^^^^^^^ @@ -1428,7 +1428,7 @@ New features and enhancements Breaking changes ^^^^^^^^^^^^^^^^ * The `tropical_nights` indice is being deprecated in favour of `tn_days_above` with ``thresh="20 degC"``. The indicator remains valid, now wrapping this new indice. -* Results of ``sdba.Grouper.apply`` for ``Grouper``s without a group (ex: ``Grouper('time')``) will contain a ``group`` singleton dimension. +* Results of ``sdba.Grouper.apply`` for ``Grouper`` without a group (ex: ``Grouper('time')``) will contain a ``group`` singleton dimension. * The `daily_freezethaw_cycles` indice is being deprecated in favour of ``multiday_temperature_swing`` with temp thresholds at 0 degC and ``window=1, op="sum"``. The indicator remains valid, now wrapping this new indice. * CMIP6 variable names have been adopted whenever possible in xclim. Changes are: diff --git a/pyproject.toml b/pyproject.toml index 1a5cfd9f2..4ef5adc8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,11 +224,13 @@ exclude = [ ] [tool.mypy] -python_version = 3.9 -show_error_codes = true +python_version = "3.9" +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +plugins = ["numpy.typing.mypy_plugin"] +strict = true warn_return_any = true +warn_unreachable = true warn_unused_configs = true -plugins = ["numpy.typing.mypy_plugin"] [[tool.mypy.overrides]] module = [ @@ -247,13 +249,18 @@ module = [ ignore_missing_imports = true [tool.pytest.ini_options] +minversion = "7.0" addopts = [ + "-ra", "--verbose", "--color=yes", "--numprocesses=0", "--maxprocesses=8", - "--dist=worksteal" + "--dist=worksteal", + "--strict-config", + "--strict-markers" ] +log_cli_level = "INFO" norecursedirs = ["docs/notebooks/*"] filterwarnings = ["ignore::UserWarning"] testpaths = [ From 2941dca1742d254b86c85ddf230ddb0fc2781357 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 17:11:38 -0400 Subject: [PATCH 17/25] adjust linting rules, run pre-commit fixes, add more error information --- docs/notebooks/benchmarks/sdba_quantile.ipynb | 2 +- pyproject.toml | 6 ++--- tests/conftest.py | 2 +- tests/test_analog.py | 2 +- tests/test_cli.py | 2 +- tests/test_locales.py | 2 +- tests/test_modules.py | 3 +-- tests/test_sdba/test_loess.py | 14 ++++++----- tests/test_units.py | 4 ++-- xclim/cli.py | 2 +- xclim/core/indicator.py | 6 ++--- xclim/core/units.py | 4 ++-- xclim/sdba/processing.py | 2 +- xclim/testing/conftest.py | 2 +- xclim/testing/utils.py | 23 ++++++++----------- 15 files changed, 37 insertions(+), 39 deletions(-) diff --git a/docs/notebooks/benchmarks/sdba_quantile.ipynb b/docs/notebooks/benchmarks/sdba_quantile.ipynb index 67823e0ab..e115bb8fa 100644 --- a/docs/notebooks/benchmarks/sdba_quantile.ipynb +++ b/docs/notebooks/benchmarks/sdba_quantile.ipynb @@ -106,7 +106,7 @@ " for size in np.arange(250, 2000 + 250, 250):\n", " da = tx.isel(time=slice(0, size))\n", " t0 = time.time()\n", - " for ii in range(num_tests):\n", + " for _i in range(num_tests):\n", " sdba.nbutils.quantile(da, **kws).compute()\n", " timed[use_fnq].append([size, time.time() - t0])\n", "\n", diff --git a/pyproject.toml b/pyproject.toml index 4ef5adc8a..c3c56a5fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -327,11 +327,11 @@ scipy = "sp" xarray = "xr" [tool.ruff.lint.isort] -known-first-party = ["xclim"] case-sensitive = true detect-same-package = false -lines-after-imports = 2 -no-lines-before = ["future", "standard-library"] +known-first-party = ["xclim"] +no-lines-before = ["future"] +required-imports = ["from __future__ import annotations"] [tool.ruff.lint.mccabe] max-complexity = 20 diff --git a/tests/conftest.py b/tests/conftest.py index dc2b25fc6..08858a306 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,10 +23,10 @@ TESTDATA_REPO_URL, default_testdata_cache, gather_testing_data, + testing_setup_warnings, ) from xclim.testing.utils import nimbus as _nimbus from xclim.testing.utils import open_dataset as _open_dataset -from xclim.testing.utils import testing_setup_warnings @pytest.fixture diff --git a/tests/test_analog.py b/tests/test_analog.py index 2608df226..6ba948871 100644 --- a/tests/test_analog.py +++ b/tests/test_analog.py @@ -256,7 +256,7 @@ def test_accuracy(self, random): out = [] n = 500 - for i in range(500): + for _i in range(500): out.append( xca.kldiv( p.rvs(n, random_state=random), q.rvs(n, random_state=random), k=k diff --git a/tests/test_cli.py b/tests/test_cli.py index 7c18a04c5..ee87df320 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -43,7 +43,7 @@ def test_indices(): runner = CliRunner() results = runner.invoke(cli, ["indices"]) - for name, ind in xclim.core.indicator.registry.items(): + for name in xclim.core.indicator.registry: assert name.lower() in results.output diff --git a/tests/test_locales.py b/tests/test_locales.py index 1aa09dcdb..41707ddc2 100644 --- a/tests/test_locales.py +++ b/tests/test_locales.py @@ -143,7 +143,7 @@ def test_xclim_translations(locale, official_indicators): continue # Both global attrs are present is_complete = {"title", "abstract"}.issubset(set(trans)) - for attrs, transattrs in zip(indcls.cf_attrs, trans["cf_attrs"]): + for _attrs, transattrs in zip(indcls.cf_attrs, trans["cf_attrs"]): if {"long_name", "description"} - set(transattrs.keys()): is_complete = False diff --git a/tests/test_modules.py b/tests/test_modules.py index 63db598a8..d2817b37b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -269,9 +269,8 @@ def test_all(self): # It's not really slow, but this is an unstable test (when it fails) and we might not want to execute it on all builds @pytest.mark.slow def test_encoding(): - import sys - import _locale + import sys # remove xclim del sys.modules["xclim"] diff --git a/tests/test_sdba/test_loess.py b/tests/test_sdba/test_loess.py index 15835014e..7fe607cd3 100644 --- a/tests/test_sdba/test_loess.py +++ b/tests/test_sdba/test_loess.py @@ -5,12 +5,14 @@ import pytest import xarray as xr -from xclim.sdba.loess import _constant_regression # noqa -from xclim.sdba.loess import _gaussian_weighting # noqa -from xclim.sdba.loess import _linear_regression # noqa -from xclim.sdba.loess import _loess_nb # noqa -from xclim.sdba.loess import _tricube_weighting # noqa -from xclim.sdba.loess import loess_smoothing +from xclim.sdba.loess import ( + _constant_regression, # noqa + _gaussian_weighting, # noqa + _linear_regression, # noqa + _loess_nb, # noqa + _tricube_weighting, # noqa + loess_smoothing, +) @pytest.mark.slow diff --git a/tests/test_units.py b/tests/test_units.py index 2f383cba4..67fa6dde8 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -253,12 +253,12 @@ def test_amount2lwethickness(snw_series): swe = amount2lwethickness(snw, out_units="mm") # FIXME: Asserting these statements shows that they are not equal - swe.attrs["standard_name"] == "lwe_thickness_of_snowfall_amount" + assert swe.attrs["standard_name"] == "lwe_thickness_of_snowfall_amount" np.testing.assert_allclose(swe, 1) snw = lwethickness2amount(swe) # FIXME: Asserting these statements shows that they are not equal - snw.attrs["standard_name"] == "snowfall_amount" + assert snw.attrs["standard_name"] == "snowfall_amount" @pytest.mark.parametrize( diff --git a/xclim/cli.py b/xclim/cli.py index 61415dd6b..f7544527c 100644 --- a/xclim/cli.py +++ b/xclim/cli.py @@ -104,7 +104,7 @@ def _process_indicator(indicator, ctx, **params): try: out = indicator(**params) except MissingVariableError as err: - raise click.BadArgumentUsage(err.args[0]) + raise click.BadArgumentUsage(err.args[0]) from err if isinstance(out, tuple): dsout = dsout.assign(**{var.name: var for var in out}) diff --git a/xclim/core/indicator.py b/xclim/core/indicator.py index 5efb09be7..cc81217ed 100644 --- a/xclim/core/indicator.py +++ b/xclim/core/indicator.py @@ -104,17 +104,17 @@ import warnings import weakref from collections import OrderedDict, defaultdict +from collections.abc import Sequence from copy import deepcopy from dataclasses import asdict, dataclass from functools import reduce from inspect import Parameter as _Parameter -from inspect import Signature +from inspect import Signature, signature from inspect import _empty as _empty_default # noqa -from inspect import signature from os import PathLike from pathlib import Path from types import ModuleType -from typing import Any, Callable, Optional, Sequence, Union +from typing import Any, Callable, Optional, Union import numpy as np import xarray diff --git a/xclim/core/units.py b/xclim/core/units.py index d3d3f2b43..51b479c27 100644 --- a/xclim/core/units.py +++ b/xclim/core/units.py @@ -1271,7 +1271,7 @@ def wrapper(*args, **kwargs): return out - setattr(wrapper, "relative_units", units_by_name) + wrapper.relative_units = units_by_name return wrapper return dec @@ -1351,7 +1351,7 @@ def wrapper(*args, **kwargs): return out - setattr(wrapper, "in_units", units_by_name) + wrapper.in_units = units_by_name return wrapper return dec diff --git a/xclim/sdba/processing.py b/xclim/sdba/processing.py index b9e76305a..77c8422a7 100644 --- a/xclim/sdba/processing.py +++ b/xclim/sdba/processing.py @@ -771,7 +771,7 @@ def stack_variables(ds: xr.Dataset, rechunk: bool = True, dim: str = "multivar") # sort to have coherent order with different datasets data_vars = sorted(ds.data_vars.items(), key=lambda e: e[0]) nvar = len(data_vars) - for i, (nm, var) in enumerate(data_vars): + for i, (_nm, var) in enumerate(data_vars): for name, attr in var.attrs.items(): attrs.setdefault(f"_{name}", [None] * nvar)[i] = attr diff --git a/xclim/testing/conftest.py b/xclim/testing/conftest.py index 3c5483561..eb0fa520f 100644 --- a/xclim/testing/conftest.py +++ b/xclim/testing/conftest.py @@ -20,10 +20,10 @@ TESTDATA_CACHE_DIR, TESTDATA_REPO_URL, gather_testing_data, + testing_setup_warnings, ) from xclim.testing.utils import nimbus as _nimbus from xclim.testing.utils import open_dataset as _open_dataset -from xclim.testing.utils import testing_setup_warnings @pytest.fixture(autouse=True, scope="session") diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index 00450c40f..12b91a0c9 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -247,7 +247,8 @@ def publish_release_notes( r":user:`([a-zA-Z0-9_.-]+)`": r"[@\1](https://github.com/\1)", } else: - raise NotImplementedError() + msg = f"Formatting style not supported: {style}" + raise NotImplementedError(msg) for search, replacement in hyperlink_replacements.items(): changes = re.sub(search, replacement, changes) @@ -573,11 +574,9 @@ def open_dataset( return _open_dataset(audit_url(dap_target, context="OPeNDAP"), **kwargs) except URLError: raise - except OSError: - raise OSError( - "OPeNDAP file not read. Verify that the service is available: %s" - % dap_target - ) + except OSError as err: + msg = f"OPeNDAP file not read. Verify that the service is available: {dap_target}" + raise OSError(msg) from err local_file = Path(cache_dir).joinpath(name) if not local_file.exists(): @@ -585,11 +584,9 @@ def open_dataset( local_file = nimbus(branch=branch, repo=repo, cache_dir=cache_dir).fetch( name ) - except OSError: - raise OSError( - "File not found locally. Verify that the testing data is available in remote: %s" - % local_file - ) + except OSError as err: + msg = f"File not found locally. Verify that the testing data is available in remote: {local_file}" + raise OSError(msg) from err try: ds = _open_dataset(local_file, **kwargs) return ds @@ -633,13 +630,13 @@ def populate_testing_data( msg = f"File `{file}` not accessible in remote repository." logging.error(msg) errored_files.append(file) - except SocketBlockedError as e: # noqa + except SocketBlockedError as err: # noqa msg = ( "Unable to access registry file online. Testing suite is being run with `--disable-socket`. " "If you intend to run tests with this option enabled, please download the file beforehand with the " "following console command: `$ xclim prefetch_testing_data`." ) - raise SocketBlockedError(msg) from e + raise SocketBlockedError(msg) from err else: logging.info("Files were downloaded successfully.") From c32cb035171307868b0bf946808a8e5b9d877a1a Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 17:19:10 -0400 Subject: [PATCH 18/25] do not reformat publish-mastodon-template.md --- .github/publish-mastodon-template.md | 1 + .pre-commit-config.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/publish-mastodon-template.md b/.github/publish-mastodon-template.md index 9f04b6e49..308374f11 100644 --- a/.github/publish-mastodon-template.md +++ b/.github/publish-mastodon-template.md @@ -2,3 +2,4 @@ New #xclim release: {{ .version }} 🎉 Latest source code available at: https://github.com/Ouranosinc/xclim/releases/tag/{{ .version }} Check out the docs for more information: https://xclim.readthedocs.io/en/stable/ + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48b1a38c0..e83c86fb1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,7 +85,7 @@ repos: rev: 0.7.17 hooks: - id: mdformat - exclude: '.github/\w+.md|docs/paper/paper.md' + exclude: '.github/\w+.md|.github/publish-mastodon-template.md|docs/paper/paper.md' - repo: https://github.com/keewis/blackdoc rev: v0.3.9 hooks: From 97167f044e5c4660d6fafc1be08fcc28b74d6020 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 17:22:01 -0400 Subject: [PATCH 19/25] use dependency grouping to reduce number of PRs opened every week --- .github/dependabot.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 826840927..176249b1c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,13 +3,19 @@ updates: - package-ecosystem: github-actions directory: / schedule: - interval: daily + interval: weekly time: '12:00' - open-pull-requests-limit: 5 + groups: + actions: + patterns: + - "*" - package-ecosystem: pip directory: / schedule: - interval: daily + interval: weekly time: '12:00' - open-pull-requests-limit: 5 + groups: + python: + patterns: + - "*" From f05325806c6559c53435b2ed44dc2b3a66869473 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 9 Sep 2024 17:28:37 -0400 Subject: [PATCH 20/25] set xfail_strict for tests --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index c3c56a5fd..7b2022550 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -272,6 +272,7 @@ markers = [ "requires_docs: mark tests that can only be run with documentation present (deselect with '-m \"not requires_docs\"')", "requires_internet: mark tests that require internet access (deselect with '-m \"not requires_internet\"')" ] +xfail_strict = true [tool.ruff] src = ["xclim"] From 2e4a42b2f90367bb8bf84ab2ea7c1af711b6a53c Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:30:09 -0400 Subject: [PATCH 21/25] adjust tests --- tests/test_run_length.py | 2 +- tests/test_sdba/test_adjustment.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_run_length.py b/tests/test_run_length.py index b0dc1501a..a109f83e6 100644 --- a/tests/test_run_length.py +++ b/tests/test_run_length.py @@ -95,7 +95,7 @@ def ufunc(request): @pytest.mark.parametrize("index", ["first", "last"]) def test_rle(ufunc, use_dask, index): if use_dask and ufunc: - pytest.xfail("rle_1d is not implemented for dask arrays.") + pytest.skip("rle_1d is not implemented for dask arrays.") values = np.zeros((10, 365, 4, 4)) time = pd.date_range("2000-01-01", periods=365, freq="D") diff --git a/tests/test_sdba/test_adjustment.py b/tests/test_sdba/test_adjustment.py index 26a771740..f7d1c47f8 100644 --- a/tests/test_sdba/test_adjustment.py +++ b/tests/test_sdba/test_adjustment.py @@ -242,6 +242,7 @@ def test_quantiles(self, series, kind, name, random): @pytest.mark.xfail( raises=ValueError, reason="This test sometimes fails due to a block/indexing error", + strict=False, ) @pytest.mark.parametrize("kind,name", [(ADDITIVE, "tas"), (MULTIPLICATIVE, "pr")]) @pytest.mark.parametrize("add_dims", [True, False]) From a65ecd10ace03e0545cb92288bc612afc2e8f221 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:53:07 -0400 Subject: [PATCH 22/25] no more verbose testing --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7b2022550..c38bdcc19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -252,7 +252,6 @@ ignore_missing_imports = true minversion = "7.0" addopts = [ "-ra", - "--verbose", "--color=yes", "--numprocesses=0", "--maxprocesses=8", From b8b9cd703f359e3e9e1c7b88177deec89c8d89cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Dupuis?= Date: Tue, 17 Sep 2024 12:12:39 -0400 Subject: [PATCH 23/25] fix test_amount2lwethickness --- tests/test_units.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_units.py b/tests/test_units.py index 67fa6dde8..158bf26a0 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -252,13 +252,11 @@ def test_amount2lwethickness(snw_series): snw = snw_series(np.ones(365), start="2019-01-01") swe = amount2lwethickness(snw, out_units="mm") - # FIXME: Asserting these statements shows that they are not equal - assert swe.attrs["standard_name"] == "lwe_thickness_of_snowfall_amount" + assert swe.attrs["standard_name"] == "lwe_thickness_of_surface_snow_amount" np.testing.assert_allclose(swe, 1) snw = lwethickness2amount(swe) - # FIXME: Asserting these statements shows that they are not equal - assert snw.attrs["standard_name"] == "snowfall_amount" + assert snw.attrs["standard_name"] == "surface_snow_amount" @pytest.mark.parametrize( From 1140b7c6e709cdf0c2bdf1c27b4f063699e44ef5 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:06:28 -0400 Subject: [PATCH 24/25] fix CHANGELOG.rst --- CHANGELOG.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5e9aad5bc..0791804e7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,14 +19,11 @@ Bug fixes ^^^^^^^^^ * Fixed a small inefficiency in ``_otc_adjust``, and the `standardize` method of `OTC/dOTC` is now applied on individual variable (:pull:`1890`, :pull:`1896`). -Breaking changes -^^^^^^^^^^^^^^^^ -* `transform` argument of `OTC/dOTC` classes (and child functions) is changed to `normalization`, and `numIterMax` is changed to `num_iter_max` in `utils.optimal_transport` (:pull:`1896`). - Breaking changes ^^^^^^^^^^^^^^^^ * `platformdirs` is no longer a direct dependency of `xclim`, but `pooch` is required to use many of the new testing functions (installable via `pip install pooch` or `pip install 'xclim[dev]'`). (:pull:`1889`). * The following previously-deprecated functions have now been removed from `xclim`: ``xclim.core.calendar.convert_calendar``, ``xclim.core.calendar.date_range``, ``xclim.core.calendar.date_range_like``, ``xclim.core.calendar.interp_calendar``, ``xclim.core.calendar.days_in_year``, ``xclim.core.calendar.datetime_to_decimal_year``. For guidance on how to migrate to alternatives, see the `version 0.50.0 Breaking changes <#v0-50-0-2024-06-17>`_. (:issue:`1010`, :pull:`1845`). +* `transform` argument of `OTC/dOTC` classes (and child functions) is changed to `normalization`, and `numIterMax` is changed to `num_iter_max` in `utils.optimal_transport` (:pull:`1896`). Internal changes ^^^^^^^^^^^^^^^^ @@ -46,6 +43,7 @@ Internal changes CI changes ^^^^^^^^^^ * The `pip` cache, `tox` environments, and the `xclim-testdata` cache are now saved between workflow runs (using `actions/cache`) to reduce the time spent installing dependencies and downloading testing data. (:pull:`1906`). + v0.52.2 (2024-09-16) -------------------- Contributors to this version: Pascal Bourgault (:user:`aulemahal`). From 4c1cd394764b0fd6b72285dd19f8c3282e1dd1f3 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:28:08 -0400 Subject: [PATCH 25/25] update CHANGELOG.rst --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0791804e7..efafee663 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -39,6 +39,12 @@ Internal changes * The codebase has been adjusted to address many `pylint`-related warnings and errors. In some cases, `casting` was used to redefine some `numpy` and `xarray` objects. (:issue:`1719`, :pull:`1881`). * ``xclim.core`` now uses absolute imports for clarity and some objects commonly used in the module have been moved to hidden submodules. (:issue:`1719`, :pull:`1881`). * ``xclim.core.indicator.Parameter`` has a new attribute ``compute_name`` while ``xclim.core.indicator.Indicator`` lost its ``_variable_mapping``. The translation from parameter (and variable) names in the indicator to the names on the compute function is handled by ``Indicator._get_compute_args``. (:pull:`1885`). +* Adopted many linting and formatting suggestions from the Scientific Python `repo-review `_ tool: (:pull:`1910`) + * Applied several linting suggestions adopted by the `scipy` community. + * Replaced `isort` with `ruff`-based import-sorting formatting. + * Added formatting for `Markdown` files. + * Added the `bugbear`, `pyupgrade` checks to the `ruff` formatter. + * Adjusted `mypy` checks to be more standardized. CI changes ^^^^^^^^^^