From d1a199785bc109863e527032c402207b793bc679 Mon Sep 17 00:00:00 2001 From: Marek Jacob <1129-b380572@users.noreply.gitlab.dkrz.de> Date: Fri, 27 Sep 2024 11:27:09 +0200 Subject: [PATCH 1/9] Enable inference when no constant forcings are used --- .../checkpoint/metadata/version_0_2_0.py | 2 +- src/anemoi/inference/runner.py | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py b/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py index 1c1c369..93dbd94 100644 --- a/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py +++ b/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py @@ -133,7 +133,7 @@ def param_level_ml_pairs(self): @property def param_step_sfc_pairs(self): - return self.request["param_step"].get("sfc", []) + return self.request.get("param_step", {}).get("sfc", []) @property def variables_with_nans(self): diff --git a/src/anemoi/inference/runner.py b/src/anemoi/inference/runner.py index 1890592..690ace4 100644 --- a/src/anemoi/inference/runner.py +++ b/src/anemoi/inference/runner.py @@ -112,7 +112,7 @@ def run( input_fields = input_fields.sel(**self.checkpoint.select) input_fields = input_fields.order_by(**self.checkpoint.order_by) - number_of_grid_points = len(input_fields[0].grid_points()[0]) + number_of_grid_points = len(input_fields[0].values) LOGGER.info("Loading input: %d fields (lagged=%d)", len(input_fields), len(self.lagged)) @@ -236,9 +236,10 @@ def run( input_tensor_numpy[:, prognostic_input_mask] = input_fields_numpy[:, prognostic_data_from_retrieved_fields_mask] - input_tensor_numpy[:, constant_from_input_mask] = input_fields_numpy[ - :, constant_data_from_retrieved_fields_mask - ] + if len(constant_from_input_mask) > 0: + input_tensor_numpy[:, constant_from_input_mask] = input_fields_numpy[ + :, constant_data_from_retrieved_fields_mask + ] constants = forcing_and_constants( source=input_fields[:1], @@ -249,13 +250,14 @@ def run( for i in range(len(self.lagged)): input_tensor_numpy[i, computed_constant_mask] = constants - for i in range(len(self.lagged)): - forcings = forcing_and_constants( - source=input_fields[:1], - param=self.checkpoint.computed_forcings, - date=start_datetime + datetime.timedelta(hours=self.lagged[i]), - ) - input_tensor_numpy[i, computed_forcing_mask] = forcings + if len(constant_from_input_mask) > 0: + for i in range(len(self.lagged)): + forcings = forcing_and_constants( + source=input_fields[:1], + param=self.checkpoint.computed_forcings, + date=start_datetime + datetime.timedelta(hours=self.lagged[i]), + ) + input_tensor_numpy[i, computed_forcing_mask] = forcings LOGGER.info("Input tensor shape: %s", input_tensor_numpy.shape) @@ -405,7 +407,8 @@ def get_most_recent_datetime(input_fields): # Update dynamic tensor for next iteration input_tensor_torch = input_tensor_torch.roll(-1, dims=1) input_tensor_torch[:, -1, :, prognostic_input_mask] = prognostic_fields - input_tensor_torch[:, -1, :, computed_forcing_mask] = forcing + if computed_forcing_mask: + input_tensor_torch[:, -1, :, computed_forcing_mask] = forcing # progress_callback(i) From 99d064493f351025d8c064eee81d90ec9922ea41 Mon Sep 17 00:00:00 2001 From: Marek Jacob <1129-b380572@users.noreply.gitlab.dkrz.de> Date: Fri, 27 Sep 2024 11:35:54 +0200 Subject: [PATCH 2/9] adjust Copyright statements --- src/anemoi/inference/checkpoint/metadata/version_0_2_0.py | 8 ++++---- src/anemoi/inference/runner.py | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py b/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py index 93dbd94..bb16138 100644 --- a/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py +++ b/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py @@ -1,10 +1,10 @@ # (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts. +# (C) Copyright 2024 Deutscher Wetterdienst. # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - +# In applying this licence, the above institution do not waive the privileges +# and immunities granted to it by virtue of its status as an intergovernmental +# organisation nor does it submit to any jurisdiction. import logging from functools import cached_property diff --git a/src/anemoi/inference/runner.py b/src/anemoi/inference/runner.py index 690ace4..21b168d 100644 --- a/src/anemoi/inference/runner.py +++ b/src/anemoi/inference/runner.py @@ -1,9 +1,10 @@ # (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts. +# (C) Copyright 2024 Deutscher Wetterdienst. # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. +# In applying this licence, the above institution do not waive the privileges +# and immunities granted to it by virtue of its status as an intergovernmental +# organisation nor does it submit to any jurisdiction. import datetime From 6a49da268118f6817e4633077ce6f38440352bb7 Mon Sep 17 00:00:00 2001 From: Marek Jacob <1129-b380572@users.noreply.gitlab.dkrz.de> Date: Fri, 27 Sep 2024 11:36:02 +0200 Subject: [PATCH 3/9] add to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31d1c9b..3094cd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Keep it human-readable, your future self will thank you! ## [Unreleased] ### Added +- Fix: Enable inference when no constant forcings are used ### Changed From fe2d21b1643f8821e583ce8eacf3c7ed52144586 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 18:40:56 +0000 Subject: [PATCH 4/9] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/astral-sh/ruff-pre-commit: v0.6.5 → v0.6.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.5...v0.6.9) - [github.com/tox-dev/pyproject-fmt: 2.2.3 → 2.2.4](https://github.com/tox-dev/pyproject-fmt/compare/2.2.3...2.2.4) - [github.com/jshwi/docsig: v0.60.1 → v0.64.0](https://github.com/jshwi/docsig/compare/v0.60.1...v0.64.0) --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ffd2c39..82ef0de 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,12 +5,12 @@ repos: - id: clear-notebooks-output name: clear-notebooks-output files: tools/.*\.ipynb$ - stages: [commit] + stages: [Nonepre-commitNone] language: python entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace additional_dependencies: [jupyter] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-yaml # Check YAML files for syntax errors only args: [--unsafe, --allow-multiple-documents] @@ -40,7 +40,7 @@ repos: - --force-single-line-imports - --profile black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.5 + rev: v0.6.9 hooks: - id: ruff # Next line if for documenation cod snippets @@ -66,11 +66,11 @@ repos: - id: docconvert args: ["numpy"] - repo: https://github.com/tox-dev/pyproject-fmt - rev: "2.2.3" + rev: "2.2.4" hooks: - id: pyproject-fmt - repo: https://github.com/jshwi/docsig # Check docstrings against function sig - rev: v0.60.1 + rev: v0.64.0 hooks: - id: docsig args: From e7fa0ab5f8a085adaee305d989421f7a3cd2d0c7 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:36:45 +0100 Subject: [PATCH 5/9] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ffd2c39..2a5d9ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: hooks: - id: ruff # Next line if for documenation cod snippets - exclude: '^[^_].*_\.py$' + exclude: '.*/[^_].*_\.py$' args: - --line-length=120 - --fix From b47aaf087c1f97888cc1353a052afef92c8283f9 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Thu, 10 Oct 2024 16:33:01 +0100 Subject: [PATCH 6/9] add link to transform --- CHANGELOG.md | 3 +++ docs/conf.py | 4 ++++ docs/index.rst | 1 + 3 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8065c1..4ca891a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,11 @@ Please add your functional changes to the appropriate section in the PR. Keep it human-readable, your future self will thank you! ## [Unreleased] + ### Added +- Add anemoi-transform link to documentation + ### Changed - Add cos_solar_zenith_angle to list of known forcings diff --git a/docs/conf.py b/docs/conf.py index 39905cd..17c48b5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -103,6 +103,10 @@ "https://anemoi-registry.readthedocs.io/en/latest/", ("../../anemoi-registry/docs/_build/html/objects.inv", None), ), + "anemoi-transform": ( + "https://anemoi-transform.readthedocs.io/en/latest/", + ("../../anemoi-transform/docs/_build/html/objects.inv", None), + ), } diff --git a/docs/index.rst b/docs/index.rst index f8cc859..990b596 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -59,6 +59,7 @@ of the *Anemoi* packages. ***************** - :ref:`anemoi-utils ` +- :ref:`anemoi-transform ` - :ref:`anemoi-datasets ` - :ref:`anemoi-models ` - :ref:`anemoi-graphs ` From 4fcdaab12eb8736a1f212afe41a57e6c3b23d9d1 Mon Sep 17 00:00:00 2001 From: Marek Jacob <1129-b380572@users.noreply.gitlab.dkrz.de> Date: Mon, 14 Oct 2024 12:05:10 +0200 Subject: [PATCH 7/9] undo the changes to the copyright statements --- src/anemoi/inference/checkpoint/metadata/version_0_2_0.py | 8 ++++---- src/anemoi/inference/runner.py | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py b/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py index bb16138..93dbd94 100644 --- a/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py +++ b/src/anemoi/inference/checkpoint/metadata/version_0_2_0.py @@ -1,10 +1,10 @@ # (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts. -# (C) Copyright 2024 Deutscher Wetterdienst. # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, the above institution do not waive the privileges -# and immunities granted to it by virtue of its status as an intergovernmental -# organisation nor does it submit to any jurisdiction. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + import logging from functools import cached_property diff --git a/src/anemoi/inference/runner.py b/src/anemoi/inference/runner.py index 21b168d..690ace4 100644 --- a/src/anemoi/inference/runner.py +++ b/src/anemoi/inference/runner.py @@ -1,10 +1,9 @@ # (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts. -# (C) Copyright 2024 Deutscher Wetterdienst. # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, the above institution do not waive the privileges -# and immunities granted to it by virtue of its status as an intergovernmental -# organisation nor does it submit to any jurisdiction. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. import datetime From 70ea07b83ee3c668cfc12c0d6e5dc9b0600ddef7 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Tue, 15 Oct 2024 17:41:03 +0100 Subject: [PATCH 8/9] Update CODEOWNERS --- .github/CODEOWNERS | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 16bb14d..74bdac0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,9 +1,6 @@ # CODEOWNERS file # Protect workflow files -/.github/ @theissenhelen @jesperdramsch @gmertes -/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes -/pyproject.toml @theissenhelen @jesperdramsch @gmertes - -# Protect package exemptions -/src/anemoi/inference/checkpoint/package_exemptions.py @gmertes @hcookie @theissenhelen @jesperdramsch +/.github/ @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb @anaprietonem @HCookie @JPXKQX @mchantry +/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb @anaprietonem @HCookie @JPXKQX @mchantry +/pyproject.toml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb @anaprietonem @HCookie @JPXKQX @mchantry From 8a28f49e53c9d4685fb5014ca757d51bbdf94bdd Mon Sep 17 00:00:00 2001 From: Harrison Cook Date: Thu, 17 Oct 2024 13:38:46 +0000 Subject: [PATCH 9/9] [fix] Issue in precommit, remove None's --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82ef0de..e01d6a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: clear-notebooks-output name: clear-notebooks-output files: tools/.*\.ipynb$ - stages: [Nonepre-commitNone] + stages: [pre-commit] language: python entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace additional_dependencies: [jupyter]