diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 562ff15..0000000 --- a/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Config file for automatic testing at travis-ci.com - -language: python -python: - - 3.8 - - 3.7 - - 3.6 - # - 3.5 - -before_install: - - sudo apt-get update - - sudo apt-get install cdo - -# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -U tox-travis - -# Command to run tests, e.g. python setup.py test -script: tox - -# Assuming you have installed the travis-ci CLI tool, after you -# create the Github repo and add it to Travis, run the -# following command to finish PyPI deployment setup: -# $ travis encrypt --add deploy.password -deploy: - provider: pypi - distributions: sdist bdist_wheel - user: larsbuntemeyer - password: - secure: PLEASE_REPLACE_ME - on: - tags: true - repo: remo-rcm/pyremo2 - python: 3.8 - -after_success: - - coveralls diff --git a/AUTHORS.rst b/AUTHORS.rst deleted file mode 100644 index 7087ea2..0000000 --- a/AUTHORS.rst +++ /dev/null @@ -1,13 +0,0 @@ -======= -Credits -======= - -Development Lead ----------------- - -* `Lars Buntemeyer `_ - -Contributors ------------- - -* `Ludwig Lierhammer `_ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index 8d6f93c..0000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,125 +0,0 @@ -.. highlight:: shell - -============ -Contributing -============ - -Contributions are welcome, and they are greatly appreciated! Every little bit -helps, and credit will always be given. - -You can contribute in many ways: - -Types of Contributions ----------------------- - -Report Bugs -~~~~~~~~~~~ - -Report bugs at https://github.com/remo-rcm/pyremo/issues. - -If you are reporting a bug, please include: - -* Your operating system name and version. -* Any details about your local setup that might be helpful in troubleshooting. -* Detailed steps to reproduce the bug. - -Fix Bugs -~~~~~~~~ - -Look through the GitHub issues for bugs. Anything tagged with "bug" and "help -wanted" is open to whoever wants to implement it. - -Implement Features -~~~~~~~~~~~~~~~~~~ - -Look through the GitHub issues for features. Anything tagged with "enhancement" -and "help wanted" is open to whoever wants to implement it. - -Write Documentation -~~~~~~~~~~~~~~~~~~~ - -The pyremo package could always use more documentation, whether as part of the -official Cordex Python Package docs, in docstrings, or even on the web in blog posts, -articles, and such. - -Submit Feedback -~~~~~~~~~~~~~~~ - -The best way to send feedback is to file an issue at https://github.com/remo-rcm/pyremo/issues. - -If you are proposing a feature: - -* Explain in detail how it would work. -* Keep the scope as narrow as possible, to make it easier to implement. -* Remember that this is a volunteer-driven project, and that contributions - are welcome :) - -Get Started! ------------- - -Ready to contribute? Here's how to set up `pyremo` for local development. - -Forking -~~~~~~~ - -You will need create your own fork of the project. This is really easy using the github -interface, just go to the `pyremo project page `_ and hit the ``Fork`` button. -From your fork, you then clone the repository to your machine:: - - git clone https://github.com/your-user-name/pyremo.git - cd pyremo - git remote add upstream https://github.com/remo-rcm/pyremo.git - - -Creating a Python Environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For the development environment, we recommend to use the conda package manager. - -- Install either `Anaconda `_ or `miniconda - `_ -- Make sure your conda is up to date (``conda update conda``) -- ``cd`` to the *pyremo* source directory - -We don't recommend to use pip installation for development since some -depdencenies (like ``cartopy`` or ``xesmf``) require pre-compiled libraries -in the backend. So the safest way to go is: - -1. Install the build dependencies -2. Build and install pyremo from source - -.. code-block:: sh - - # Create and activate the build environment - conda create -c conda-forge -n pyremo-tests python=3.9 - - conda env update -f ci/requirements/environment.yml - - conda activate pyremo-tests - - # Build and install pyremo in editable mode - pip install -e . - -At this point you should be able to import *pyremo* from your locally -built version: - -.. code-block:: sh - - $ python # start an interpreter - >>> import pyremo - >>> pyremo.__version__ - -The nice thing about the *editable* mode (that's the ``-e`` flag in the pip install command) is -that you can edit the code directly in the package and use it without having to reinstall -the package. If you work a lot in Jupyter notebooks for development, you should check out -the autoreload magic, e.g., add a cell in the top of your notebook containing: - -.. code-block:: sh - - %load_ext autoreload - %autoreload 2 - -This will allow you to edit the *pyremo* source code and use it directly in the notebook -without having to restart the kernel. - -See the full conda docs `here `__. diff --git a/HISTORY.rst b/HISTORY.rst deleted file mode 100644 index b5a4202..0000000 --- a/HISTORY.rst +++ /dev/null @@ -1,8 +0,0 @@ -======= -History -======= - -0.1.0 (2020-07-23) ------------------- - -* First release on PyPI. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 965b2dd..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,11 +0,0 @@ -include AUTHORS.rst -include CONTRIBUTING.rst -include HISTORY.rst -include LICENSE -include README.rst - -recursive-include tests * -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] - -recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index 8fd84a1..515a38e 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -26,6 +26,5 @@ dependencies: - cmor - cdo - python-cdo - - py-cordex>=0.4 -# - pip: -# - git+https://github.com/euro-cordex/py-cordex.git # always use master for now + - pip: + - git+https://github.com/euro-cordex/py-cordex.git # always use master for now diff --git a/docs/authors.rst b/docs/authors.rst index e122f91..7087ea2 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -1 +1,13 @@ -.. include:: ../AUTHORS.rst +======= +Credits +======= + +Development Lead +---------------- + +* `Lars Buntemeyer `_ + +Contributors +------------ + +* `Ludwig Lierhammer `_ diff --git a/docs/contributing.rst b/docs/contributing.rst index e582053..8d6f93c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -1 +1,125 @@ -.. include:: ../CONTRIBUTING.rst +.. highlight:: shell + +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/remo-rcm/pyremo/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +The pyremo package could always use more documentation, whether as part of the +official Cordex Python Package docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/remo-rcm/pyremo/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `pyremo` for local development. + +Forking +~~~~~~~ + +You will need create your own fork of the project. This is really easy using the github +interface, just go to the `pyremo project page `_ and hit the ``Fork`` button. +From your fork, you then clone the repository to your machine:: + + git clone https://github.com/your-user-name/pyremo.git + cd pyremo + git remote add upstream https://github.com/remo-rcm/pyremo.git + + +Creating a Python Environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For the development environment, we recommend to use the conda package manager. + +- Install either `Anaconda `_ or `miniconda + `_ +- Make sure your conda is up to date (``conda update conda``) +- ``cd`` to the *pyremo* source directory + +We don't recommend to use pip installation for development since some +depdencenies (like ``cartopy`` or ``xesmf``) require pre-compiled libraries +in the backend. So the safest way to go is: + +1. Install the build dependencies +2. Build and install pyremo from source + +.. code-block:: sh + + # Create and activate the build environment + conda create -c conda-forge -n pyremo-tests python=3.9 + + conda env update -f ci/requirements/environment.yml + + conda activate pyremo-tests + + # Build and install pyremo in editable mode + pip install -e . + +At this point you should be able to import *pyremo* from your locally +built version: + +.. code-block:: sh + + $ python # start an interpreter + >>> import pyremo + >>> pyremo.__version__ + +The nice thing about the *editable* mode (that's the ``-e`` flag in the pip install command) is +that you can edit the code directly in the package and use it without having to reinstall +the package. If you work a lot in Jupyter notebooks for development, you should check out +the autoreload magic, e.g., add a cell in the top of your notebook containing: + +.. code-block:: sh + + %load_ext autoreload + %autoreload 2 + +This will allow you to edit the *pyremo* source code and use it directly in the notebook +without having to restart the kernel. + +See the full conda docs `here `__. diff --git a/docs/history.rst b/docs/history.rst deleted file mode 100644 index 2506499..0000000 --- a/docs/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../HISTORY.rst diff --git a/docs/whats_new.rst b/docs/whats_new.rst index cdb6967..200f3e8 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -8,12 +8,13 @@ What's New import pyremo -v0.5.1 (Unreleased) +v0.6.0 (Unreleased) ------------------- Internal Changes ~~~~~~~~~~~~~~~~ +- Refactoring of core modules, now using `domain_id` keyword instead of `short_name` (:pull:`125`). - Renamed ``master`` to ``main``. v0.5.1 (24 May 2023) diff --git a/pyremo/__init__.py b/pyremo/__init__.py index 09c9b09..15f0445 100644 --- a/pyremo/__init__.py +++ b/pyremo/__init__.py @@ -1,11 +1,10 @@ import pkg_resources -from . import physics, tutorial -from .core import codes, data, remo_ds -from .core.cal import parse_absolute_time, parse_dates -from .core.conventions import output_pattern -from .core.domain import domain_info, magic_number, magic_numbers, remo_domain -from .core.remo_ds import open_remo_dataset, preprocess, update_meta_info +from . import codes, data, physics, remo_ds, tutorial +from .cal import parse_absolute_time, parse_dates +from .conventions import output_pattern +from .domain import domain_info, magic_number, magic_numbers, remo_domain +from .remo_ds import open_remo_dataset, preprocess, update_meta_info from .tables import domains, vc try: diff --git a/pyremo/core/cal.py b/pyremo/cal.py similarity index 100% rename from pyremo/core/cal.py rename to pyremo/cal.py diff --git a/pyremo/cli.py b/pyremo/cli.py index bdb1c99..f39b608 100644 --- a/pyremo/cli.py +++ b/pyremo/cli.py @@ -1,12 +1,45 @@ """Console script for pyremo.""" import argparse import os -import sys +from os import path as op +from pathlib import Path -from dask.distributed import Client +import xarray as xr + +import pyremo as pr from .core import cli as core_cli +soil_default = [ + "TSL", + "TSN", + "TD3", + "TD4", + "TD5", + "GLAC", + "TD", + "TDCL", + "WS", + "WL", + "SN", +] + +fx_default = [ + "FIB", + "BLA", + "AZ0", + "ALB", + "VGRAT", + "VAROR", + "VLT", + "FOREST", + "FAO", + "WSMX", + "BETA", + "WMINLOK", + "WMAXLOK", +] + def create_parser(): """Creates parser for command line tool.""" @@ -44,6 +77,94 @@ def add_variables(): return 0 +def encode(ds): + # we have to set the encoding here explicitly, otherwise xarray.to_netcdf will + # encode missing values by NaN, which will crash REMO... + fillvars = ["TSW", "SEAICE", "TSI"] + for var in ds.data_vars: + if var in fillvars: + ds[var].encoding["_FillValue"] = 1.0e20 + else: + ds[var].encoding["_FillValue"] = None + for c in ds.coords: + ds[c].encoding["_FillValue"] = None + return ds + + +def get_output_filename(target): + path = Path(target) + return op.join(path.parent, path.stem + "_replaced" + path.suffix) + + +def replace_vars(target, source, vars, surflib=None, static=None, overwrite=False): + tds = xr.open_dataset(target) + sds = xr.open_dataset(source) + tds = tds.merge(sds[vars], compat="override", join="override") + if surflib is not None: + surflib = pr.data.surflib(surflib) + tds = tds.merge(surflib[static], compat="override", join="override") + if overwrite is True: + fname = target + else: + fname = get_output_filename(target) + print(f"writing to {fname}") + encode(tds).to_netcdf(fname) + return fname + + +def replace_parser(): + parser = argparse.ArgumentParser( + description="Replace variables in target files with data from source file. This can be used to, e.g., add soil variables from a REMO output file to a forcing file." + ) + parser.add_argument("target", metavar="target", help="target file") + parser.add_argument("source", metavar="source", help="source file") + parser.add_argument( + "-v", + "--variables", + dest="variables", + nargs="+", + help="list of variables to add or replace (default = {})".format(soil_default), + default=soil_default, + ) + parser.add_argument( + "-s", + "--surflib", + dest="surflib", + default=None, + help="domain used for surface library", + ) + parser.add_argument( + "-fx", + "--static", + dest="static", + nargs="+", + help="list of variables to add or replace from soil library (default = {})".format( + fx_default + ), + default=fx_default, + ) + parser.add_argument( + "-f", + "--overwrite", + dest="overwrite", + action="store_true", + default=False, + help="overwrite target file with new variables", + ) + return parser + + +def replace_variables(args): + return replace_vars( + args.target, + args.source, + args.variables, + surflib=args.surflib, + static=args.static, + overwrite=args.overwrite, + ) + + def main_analysis(): import xarray as xr @@ -230,8 +351,3 @@ def main_analysis(): eobs_pr_season_plot.savefig(os.path.join(output_path, "PR_REMO-EOBS.png")) return - - -if __name__ == "__main__": - with Client() as client: - sys.exit(main()) diff --git a/pyremo/cmor/utils.py b/pyremo/cmor/utils.py index 33f403f..5b1bc06 100644 --- a/pyremo/cmor/utils.py +++ b/pyremo/cmor/utils.py @@ -4,7 +4,7 @@ import cordex as cx import xarray as xr -from ..core import codes +from .. import codes def _get_varinfo(name): diff --git a/pyremo/core/codes.py b/pyremo/codes.py similarity index 99% rename from pyremo/core/codes.py rename to pyremo/codes.py index f553567..163b4ed 100644 --- a/pyremo/core/codes.py +++ b/pyremo/codes.py @@ -45,7 +45,7 @@ import numpy as np import pandas as pd -from ..tables import codes +from .tables import codes # table = pd.concat([table for name, table in code_table.items()]) diff --git a/pyremo/core/conventions.py b/pyremo/conventions.py similarity index 100% rename from pyremo/core/conventions.py rename to pyremo/conventions.py diff --git a/pyremo/core/data.py b/pyremo/data.py similarity index 100% rename from pyremo/core/data.py rename to pyremo/data.py diff --git a/pyremo/core/domain.py b/pyremo/domain.py similarity index 84% rename from pyremo/core/domain.py rename to pyremo/domain.py index 0fdaf93..9b7c4ee 100644 --- a/pyremo/core/domain.py +++ b/pyremo/domain.py @@ -35,15 +35,15 @@ import cordex as cx import pandas as pd -from ..tables import domains +from .tables import domains -def remo_domain(short_name, **kwargs): +def remo_domain(domain_id, **kwargs): """Creates an xarray dataset containg the domain grid definitions. Parameters ---------- - short_name: + domain_id: Name of the Cordex Domain. dummy : str or logical Name of dummy field, if dummy=topo, the cdo topo operator will be @@ -56,10 +56,10 @@ def remo_domain(short_name, **kwargs): Dataset containing the coordinates. """ - return cx.cordex_domain(short_name, tables=domains.table, **kwargs) + return cx.cordex_domain(domain_id, tables=domains.table, **kwargs) -def domain_info(short_name): +def domain_info(domain_id): """Returns a dictionary containg the domain grid definitions. Returns a dictionary with grid information according to the @@ -67,7 +67,7 @@ def domain_info(short_name): Parameters ---------- - short_name: + domain_id: Name of the Cordex Domain. Returns @@ -76,20 +76,7 @@ def domain_info(short_name): Dictionary containing the grid information. """ - return cx.domain_info(short_name, tables=domains.table) - - -def table(name): - """Top level function that returns a CORDEX table. - - Args: - name (str): name of the CORDEX table. - - Returns: - table (DataFrame): Cordex table. - - """ - return tables[name] + return cx.domain_info(domain_id, tables=domains.table) def magic_number(n=1, m=0, o=0): diff --git a/pyremo/prsint/cli.py b/pyremo/prsint/cli.py index 20b157a..fef80ad 100644 --- a/pyremo/prsint/cli.py +++ b/pyremo/prsint/cli.py @@ -3,9 +3,9 @@ import numpy as np import xarray as xr -from ..core import codes -from ..core import remo_ds as rds -from ..core.utilities import encode +from .. import codes +from .. import remo_ds as rds +from ..utils import encode from . import _defaults as dflt from .core import pressure_interpolation diff --git a/pyremo/core/remo_ds.py b/pyremo/remo_ds.py similarity index 100% rename from pyremo/core/remo_ds.py rename to pyremo/remo_ds.py diff --git a/pyremo/tables/_resources.py b/pyremo/tables/_resources.py index 63708f9..9e3b0e1 100644 --- a/pyremo/tables/_resources.py +++ b/pyremo/tables/_resources.py @@ -11,9 +11,7 @@ # The remote data is on Github base_url=base_url + "domains/", registry={ - "cordex-domains.csv": None, - "aux-domains.csv": None, - "cordex-fps.csv": None, + "domains.csv": None, }, ) @@ -63,14 +61,10 @@ def read_remote_table(name, resource, **kwargs): return pd.read_csv(fname, **kwargs) -def read_domain_table(name): - return read_remote_table(name, resource=DOMAIN_RESOURCE, index_col="short_name") - - def read_remo_domain_tables(): resource = DOMAIN_RESOURCE return { - table.split(".")[0]: read_remote_table(table, resource, index_col="short_name") + table.split(".")[0]: read_remote_table(table, resource, index_col="domain_id") for table in resource.registry.keys() } diff --git a/pyremo/core/utilities.py b/pyremo/utils.py similarity index 100% rename from pyremo/core/utilities.py rename to pyremo/utils.py diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 33769ba..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -xarray -pooch -py-cordex>=0.2.0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 66a43e7..a1ed32c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import datetime as dt import pyremo as pr -from pyremo.core import cal +from pyremo import cal def test_absolute_calendar(): diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 8a50f92..0000000 --- a/tox.ini +++ /dev/null @@ -1,29 +0,0 @@ -[tox] -envlist = py35, py36, py37, py38, flake8 - -[travis] -python = - 3.8: py38 - 3.7: py37 - 3.6: py36 - 3.5: py35 - -[testenv:flake8] -basepython = python -deps = flake8 -commands = flake8 pyremo tests - -[testenv] -passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH -setenv = - PYTHONPATH = {toxinidir} -deps = - -r{toxinidir}/requirements_dev.txt -; If you want to make tox run the tests with the same versions, create a -; requirements.txt with the pinned versions and uncomment the following line: -; -r{toxinidir}/requirements.txt -commands = - pip install -U pip - pytest --basetemp={envtmpdir} - coverage run --source=pyremo setup.py test - coveralls