From d81cdbd1ec04de7d6fac74f2533050f9a16f6ac2 Mon Sep 17 00:00:00 2001 From: Sam Green Date: Wed, 10 Jul 2024 09:59:13 +1000 Subject: [PATCH 01/16] Update mopper-conda.yaml to fix python version --- .github/workflows/mopper-conda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index c232518..cbd7984 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -22,7 +22,7 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - name: Add conda to system path run: | # $CONDA is an environment variable pointing to the root of the miniconda directory From d47081806ed3de27533365688eaf2ffb5ae0cba7 Mon Sep 17 00:00:00 2001 From: Sam Green Date: Wed, 10 Jul 2024 10:02:32 +1000 Subject: [PATCH 02/16] Update mopper-conda.yaml to fix conda file --- .github/workflows/mopper-conda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index cbd7984..d5868d9 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -29,7 +29,7 @@ jobs: echo $CONDA/bin >> $GITHUB_PATH - name: Install dependencies run: | - conda env update --file conda/environment.yml --name base + conda env update --file conda/meta.yml --name base - name: Lint with flake8 run: | conda install flake8 From a16bfee3a20a5269216632fd48cb33a532903802 Mon Sep 17 00:00:00 2001 From: Sam Green Date: Wed, 10 Jul 2024 10:03:41 +1000 Subject: [PATCH 03/16] Update mopper-conda.yaml --- .github/workflows/mopper-conda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index d5868d9..a8d26a6 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -29,7 +29,7 @@ jobs: echo $CONDA/bin >> $GITHUB_PATH - name: Install dependencies run: | - conda env update --file conda/meta.yml --name base + conda env update --file conda/meta.yaml --name base - name: Lint with flake8 run: | conda install flake8 From 6583c895ca2db0251913330204350628d0a0674d Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Wed, 10 Jul 2024 10:14:43 +1000 Subject: [PATCH 04/16] removed extra line from docs --- docs/gettingstarted.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 666e89a..2e7181e 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -56,7 +56,7 @@ Step2: Set up the working environment *mop -c setup* .. code-block:: console -https://climate-cms.org/posts/2023-05-31-vscode-are.html + $ mop -c exp_conf.yaml setup Simulation to process: cy286 Setting environment and creating working directory From b6a424100fdf1d87f3e15cccc07defe22dd9b4ab Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Wed, 10 Jul 2024 13:28:59 +1000 Subject: [PATCH 05/16] attempt to fix meta.yaml --- conda/meta.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index f20a79a..6a68a4f 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,7 +1,6 @@ -{% set version = "1.0.0" %} package: name: mopper - version: {{ version }} + version: 1.0.0 #source: # path: ./ From 21d5ecfde8dbd68c55eb4bf7ba315af0dd3f5998 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Wed, 10 Jul 2024 13:40:31 +1000 Subject: [PATCH 06/16] attempt to fix meta.yaml 2 --- conda/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index 6a68a4f..05d58fc 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -8,7 +8,7 @@ package: source: #url: https://github.com/ACCESS-Hive/ACCESS-MOPPeR/archive/refs/tags/{{version}}.tar.gz git_url: https://github.com/ACCESS-Hive/ACCESS-MOPPeR.git - git_rev: {{ version }} + git_rev: "{{ version }}" git_depth: 1 # (Defaults to -1/not shallow) build: From 79983531d461e83889b38a7e3bf2eec0d1f1eaac Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 08:54:25 +1000 Subject: [PATCH 07/16] fixed issues hihglighted by flake --- .github/workflows/mopper-conda.yaml | 1 + src/mopdb/mopdb_utils.py | 2 +- src/mopper/calculations.py | 5 +++-- src/mopper/setup_utils.py | 8 ++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index a8d26a6..f776b8f 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -14,6 +14,7 @@ on: jobs: build-linux: runs-on: ubuntu-latest + timeout-minutes: 60 strategy: max-parallel: 5 diff --git a/src/mopdb/mopdb_utils.py b/src/mopdb/mopdb_utils.py index b9875c7..32ceb81 100644 --- a/src/mopdb/mopdb_utils.py +++ b/src/mopdb/mopdb_utils.py @@ -663,7 +663,7 @@ def read_map(fname, alias): notes = row[16] else: notes = row[15] - if alias is '': + if alias == '': alias = fname.replace(".csv","") var_list.append(row[:11] + [notes, alias]) return var_list diff --git a/src/mopper/calculations.py b/src/mopper/calculations.py index 1adf216..c87213a 100644 --- a/src/mopper/calculations.py +++ b/src/mopper/calculations.py @@ -950,7 +950,7 @@ def tos_3hr(var, landfrac): vout : Xarray dataset """ - v = tos_degC(var) + v = K_degC(var) vout = xr.zeros_like(var) t = len(var.time) @@ -1346,6 +1346,7 @@ def get_basin_mask(ctx, lat, lon): basin_mask: DataArray basin_mask(lat,lon) """ + var_log = logging.getLogger(ctx.obj['var_log']) coords = ['t', 't'] if 'xu' in lon: coords[0] = 'u' @@ -1443,6 +1444,6 @@ def calc_depositions(ctx, var, weight=None): varlist.append(v0) if weight is None: weight = 0.05844 - deps = sum_vars(varlist) * mole_weight + deps = sum_vars(varlist) * weight return deps diff --git a/src/mopper/setup_utils.py b/src/mopper/setup_utils.py index e0341fa..192f642 100755 --- a/src/mopper/setup_utils.py +++ b/src/mopper/setup_utils.py @@ -44,6 +44,7 @@ from json.decoder import JSONDecodeError from mopdb.mopdb_utils import query +from mopdb.cmip_utils import fix_years def write_var_map(outpath, table, matches): @@ -166,7 +167,7 @@ def find_custom_tables(ctx): mop_log = logging.getLogger('mop_log') tables = [] path = ctx.obj['tables_path'] - tables = ctx.obj['tables_path'].rglob("*_*.json") + table_files = ctx.obj['tables_path'].rglob("*_*.json") for f in table_files: f = str(f).replace(".json", "") tables.append(f) @@ -626,9 +627,8 @@ def define_files(ctx, cursor, opts, mp): if mp['years'] != 'all' and ctx.obj['dreq_years']: exp_start, exp_end = fix_years(mp['years'], exp_start[:4], exp_end[:4]) if exp_start is None: - mop_log.info("Years requested for variable are outside specified") - mop_log.info((f"period: {table_id}, {var},", - f"{match['tstart']}, {match['tend']}")) + mop_log.info(f"""Years requested for variable are outside + specified period: {mp['years']}""") return tstep_dict = {'10min': ['minutes=10', 'minutes=5'], '30min': ['minutes=30', 'minutes=15'], From 57894bd6e9c31e2ffde041350b766182a1424ecb Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 09:13:05 +1000 Subject: [PATCH 08/16] fixing conda package install in actions --- .github/workflows/mopper-conda.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index f776b8f..64c4906 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -38,9 +38,6 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - # - name: Install package - run: | - conda build conda/meta.yaml - name: Test with pytest run: | conda install pytest coverage codecov From f044f04810c48c505b0626773fe93aa99dc83c51 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 09:30:03 +1000 Subject: [PATCH 09/16] fixing conda package install in actions 2 --- .github/workflows/mopper-conda.yaml | 2 +- env.yaml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 env.yaml diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index 64c4906..bed7062 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -30,7 +30,7 @@ jobs: echo $CONDA/bin >> $GITHUB_PATH - name: Install dependencies run: | - conda env update --file conda/meta.yaml --name base + conda env update --file env.yaml --name base - name: Lint with flake8 run: | conda install flake8 diff --git a/env.yaml b/env.yaml new file mode 100644 index 0000000..64e2483 --- /dev/null +++ b/env.yaml @@ -0,0 +1,7 @@ +name: mopenv +dependencies: + - click + - cmor + - xarray + - numpy + - pyyaml From 7b3460e85bf3739af5c6e7d3f2fd055ba2732e3d Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 09:34:10 +1000 Subject: [PATCH 10/16] fixing conda package install in actions 3 --- env.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/env.yaml b/env.yaml index 64e2483..10266f8 100644 --- a/env.yaml +++ b/env.yaml @@ -1,4 +1,6 @@ name: mopenv +channels: + - conda-forge dependencies: - click - cmor From e6b37e50096bbb3da99e6770a9133b4c1511eace Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 09:43:21 +1000 Subject: [PATCH 11/16] fixing conda package install in actions 4 --- .github/workflows/mopper-conda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index bed7062..f32f3a4 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -33,7 +33,7 @@ jobs: conda env update --file env.yaml --name base - name: Lint with flake8 run: | - conda install flake8 + conda install flake8 --solver classic # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide From 3c02c32fdd0b9c8bed8b2ccda34f1e1aeefedf49 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 09:46:23 +1000 Subject: [PATCH 12/16] fixing conda package install in actions 5 --- .github/workflows/mopper-conda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index f32f3a4..5ff395e 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -40,7 +40,7 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - conda install pytest coverage codecov + conda install pytest coverage codecov --solver classic conda run python -m pytest conda run coverage run --source src -m py.test - name: Upload to codecov From a36cd1395f20a899afdc42dfcfc0691030a1235c Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 10:07:14 +1000 Subject: [PATCH 13/16] fixing conda package install in actions 6 --- .github/workflows/mopper-conda.yaml | 3 +++ env.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index 5ff395e..1b005b5 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -38,6 +38,9 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: install package + source activate base + pip install ./ - name: Test with pytest run: | conda install pytest coverage codecov --solver classic diff --git a/env.yaml b/env.yaml index 10266f8..6e46467 100644 --- a/env.yaml +++ b/env.yaml @@ -7,3 +7,4 @@ dependencies: - xarray - numpy - pyyaml + - dask From a1f393e7e63dd55dda84d3cb41fcfc7057ded9e9 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 10:11:17 +1000 Subject: [PATCH 14/16] fixing conda package install in actions 7 --- .github/workflows/mopper-conda.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mopper-conda.yaml b/.github/workflows/mopper-conda.yaml index 1b005b5..d08666f 100644 --- a/.github/workflows/mopper-conda.yaml +++ b/.github/workflows/mopper-conda.yaml @@ -39,6 +39,7 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: install package + run: | source activate base pip install ./ - name: Test with pytest From 9a746c2af438c566f886fdc2db91a99eae1cfa23 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 10:20:06 +1000 Subject: [PATCH 15/16] issue #155 --- src/mopper/calculations.py | 2 +- src/mopper/mop_utils.py | 2 +- src/mopper/setup_utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mopper/calculations.py b/src/mopper/calculations.py index c87213a..d217aef 100644 --- a/src/mopper/calculations.py +++ b/src/mopper/calculations.py @@ -39,7 +39,7 @@ import dask import logging -from importlib_resources import files as import_files +from importlib.resources import files as import_files from mopper.setup_utils import read_yaml # Global Variables diff --git a/src/mopper/mop_utils.py b/src/mopper/mop_utils.py index cdb78db..6017b68 100755 --- a/src/mopper/mop_utils.py +++ b/src/mopper/mop_utils.py @@ -40,7 +40,7 @@ from mopper.calculations import * from mopper.setup_utils import read_yaml -from importlib_resources import files as import_files +from importlib.resources import files as import_files def config_log(debug, path, stream_level=logging.WARNING): diff --git a/src/mopper/setup_utils.py b/src/mopper/setup_utils.py index 192f642..be5cae0 100755 --- a/src/mopper/setup_utils.py +++ b/src/mopper/setup_utils.py @@ -40,7 +40,7 @@ from collections import OrderedDict from datetime import datetime#, timedelta from dateutil.relativedelta import relativedelta -from importlib_resources import files as import_files +from importlib.resources import files as import_files from json.decoder import JSONDecodeError from mopdb.mopdb_utils import query From 357e4e9df24767541a5b85eb373b9c7ab9eb5488 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Thu, 11 Jul 2024 10:23:55 +1000 Subject: [PATCH 16/16] fixed import --- src/mopper/setup_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mopper/setup_utils.py b/src/mopper/setup_utils.py index be5cae0..ef2c1ec 100755 --- a/src/mopper/setup_utils.py +++ b/src/mopper/setup_utils.py @@ -44,7 +44,7 @@ from json.decoder import JSONDecodeError from mopdb.mopdb_utils import query -from mopdb.cmip_utils import fix_years +from mopper.cmip_utils import fix_years def write_var_map(outpath, table, matches):