Pooch integration, mark online tests, better handle online-dependent …

…documentation, distributed testing (#212) ### What kind of change does this PR introduce? * Marks several tests reliant on external services (`online`). * Better typing in call signatures. * Removes several unused imports. * Added `pytest-xdist` to the development dependencies. * Adding in a `pooch` registry for verifying file integrity when fetching testing data. ### Does this PR introduce a breaking change? Yes. `pytest-xdist` is now a development dependency. By default, it is not being used, but it can be enabled by providing positional arguments to the `pytest` call. ### Other information: I've copied and slightly improved upon the new testing data system currently in #175. I'll handle the merge of the changes to there, but this current system is nearing finalization. The last thing that I'm waiting on are some changes from @RondeauG to `xhydro-testdata` so that I can make a dated tag that will become the default testing data branch for `xhydro` (updated whenever there are additions to `xhydro-testdata`). This will help prevent breakages from a moving `main` branch over there.
hydrologie · Oct 16, 2024 · 9057e1a · 9057e1a
2 parents a665f38 + 0e22726
commit 9057e1a
Show file tree

Hide file tree

Showing 26 changed files with 1,068 additions and 1,609 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -4,14 +4,22 @@ Changelog
 
 v0.5.0 (unreleased)
 -------------------
-Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`).
+Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`), Trevor James Smith (:user:`Zeitsperre`).
+
+Breaking changes
+^^^^^^^^^^^^^^^^
+* The `xhydro` testing utilities have been rewritten to use `pooch` for downloading and caching testing datasets from `hydrologie/xhydro-testdata`. (:pull:`212`).
+* The `xhydro` testing utilities now require `pytest-xdist` as a development dependency. (:pull:`212`).
 
 Internal changes
 ^^^^^^^^^^^^^^^^
 * Updated the notebooks to reduce the load on ReadTheDocs. (:pull:`211`).
+* Tests can now be run using the `pytest-xdist` plugin for distributed testing. See the `pytest-xdist documentation <https://pytest-xdist.readthedocs.io/en/stable/>`_ for more information. (:pull:`212`).
+* Several tests reliant on online servers and services have been marked as `online` to prevent them from running in contexts where internet access is limited. (:pull:`212`).
+* Many function docstrings and type hints have been updated for accuracy and precision. (:pull:`212`).
 
-v.4.0 (2024-10-04)
-------------------
+v0.4.0 (2024-10-04)
+-------------------
 Contributors to this version: Trevor James Smith (:user:`Zeitsperre`), Gabriel Rondeau-Genesse (:user:`RondeauG`), Thomas-Charles Fortier Filion (:user:`TC-FF`).
 
 New features and enhancements

diff --git a/docs/notebooks/climate_change.ipynb b/docs/notebooks/climate_change.ipynb
@@ -19,51 +19,31 @@
     "# Hide INFO-level logs\n",
     "import logging\n",
     "from pathlib import Path\n",
-    "from zipfile import ZipFile\n",
     "\n",
-    "import hvplot.xarray\n",
+    "import hvplot.xarray  # noqa\n",
     "import numpy as np\n",
     "import pooch\n",
     "import xarray as xr\n",
     "import xclim\n",
     "\n",
     "import xhydro as xh\n",
+    "from xhydro.testing.helpers import deveraux\n",
     "\n",
     "logger = logging.getLogger()\n",
     "logger.setLevel(logging.CRITICAL)\n",
     "\n",
-    "# This notebook will use data from the 2022 edition of the Hydrological Atlas of Southern Quebec, which can be accessed from the xhydro-testdata repository.\n",
-    "# They cover 2 stations: ABIT00057 and ABIT00058\n",
-    "GITHUB_URL = \"https://github.com/hydrologie/xhydro-testdata\"\n",
-    "BRANCH_OR_COMMIT_HASH = \"main\"\n",
+    "D = deveraux()\n",
     "\n",
     "# Streamflow file (1 file - Hydrotel driven by BCC-CSM-1.1(m))\n",
-    "streamflow_file = pooch.retrieve(\n",
-    "    url=f\"{GITHUB_URL}/raw/{BRANCH_OR_COMMIT_HASH}/data/cc_indicators/streamflow_BCC-CSM1.1-m_rcp45.nc\",\n",
-    "    known_hash=\"md5:0ac83a4ee9dceecda68ac1ee542f50de\",\n",
+    "streamflow_file = D.fetch(\n",
+    "    \"cc_indicators/streamflow_BCC-CSM1.1-m_rcp45.nc\",\n",
     ")\n",
     "\n",
     "# Reference QMOYAN (6 platforms)\n",
-    "ref_zip = pooch.retrieve(\n",
-    "    url=f\"{GITHUB_URL}/raw/{BRANCH_OR_COMMIT_HASH}/data/cc_indicators/reference.zip\",\n",
-    "    known_hash=\"md5:192544f3a081375a81d423e08038d32a\",\n",
-    ")\n",
-    "directory_to_extract_to = Path(ref_zip).parent\n",
-    "with ZipFile(ref_zip, \"r\") as ziploc:\n",
-    "    ziploc.extractall(directory_to_extract_to)\n",
-    "    files = ziploc.namelist()\n",
-    "reference_files = [directory_to_extract_to / f for f in files]\n",
+    "reference_files = D.fetch(\"cc_indicators/reference.zip\", pooch.Unzip())\n",
     "\n",
     "# QMOYAN deltas (63 simulations x 6 platforms)\n",
-    "deltas_zip = pooch.retrieve(\n",
-    "    url=f\"{GITHUB_URL}/raw/{BRANCH_OR_COMMIT_HASH}/data/cc_indicators/deltas.zip\",\n",
-    "    known_hash=\"md5:ce6371e073e5324f9ade385c1c03e7eb\",\n",
-    ")\n",
-    "directory_to_extract_to = Path(deltas_zip).parent\n",
-    "with ZipFile(deltas_zip, \"r\") as ziploc:\n",
-    "    ziploc.extractall(directory_to_extract_to)\n",
-    "    files = ziploc.namelist()\n",
-    "deltas_files = [directory_to_extract_to / f for f in files]"
+    "deltas_files = D.fetch(\"cc_indicators/deltas.zip\", pooch.Unzip())"
    ]
   },
   {
@@ -647,7 +627,7 @@
     "delta_dist = delta_dist.rename({\"sample\": \"realization\"})  # xclim compatibility\n",
     "ens_stats_2 = xh.cc.ensemble_stats(delta_dist, statistics)\n",
     "\n",
-    "# Interquartile range\n",
+    "# Inter-quartile range\n",
     "ens_stats_2[\"QMOYAN_iqr\"] = ens_stats_2[\"QMOYAN\"].sel(percentiles=75) - ens_stats_2[\n",
     "    \"QMOYAN\"\n",
     "].sel(percentiles=25)\n",

diff --git a/docs/notebooks/gis.ipynb b/docs/notebooks/gis.ipynb
diff --git a/docs/notebooks/hydrological_modelling.ipynb b/docs/notebooks/hydrological_modelling.ipynb
@@ -36,7 +36,6 @@
     "\n",
     "# This function can be called to get a list of the keys for a given model, as well as its documentation.\n",
     "inputs, docs = xhm.get_hydrological_model_inputs(\"Hydrotel\", required_only=False)\n",
-    "\n",
     "inputs"
    ]
   },
@@ -107,18 +106,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import pooch\n",
     "import xarray as xr\n",
     "\n",
+    "from xhydro.testing.helpers import deveraux\n",
+    "\n",
+    "D = deveraux()\n",
+    "\n",
     "# This notebook will use ERA5 data for a small watershed in Eastern Quebec, along with faked elevation data.\n",
-    "GITHUB_URL = \"https://github.com/hydrologie/xhydro-testdata\"\n",
-    "BRANCH_OR_COMMIT_HASH = \"main\"\n",
     "\n",
     "# Streamflow file (1 file - Hydrotel driven by BCC-CSM-1.1(m))\n",
-    "meteo_file = pooch.retrieve(\n",
-    "    url=f\"{GITHUB_URL}/raw/{BRANCH_OR_COMMIT_HASH}/data/hydro_modelling/ERA5_testdata.nc\",\n",
-    "    known_hash=\"md5:784db17edf68d9e38c3e9d0aa18297a6\",\n",
-    ")\n",
+    "meteo_file = D.fetch(\"hydro_modelling/ERA5_testdata.nc\")\n",
     "ds = xr.open_dataset(meteo_file)\n",
     "ds"
    ]
@@ -236,7 +233,7 @@
     "# Prepare the model configuration options\n",
     "model_config = {\n",
     "    \"model_name\": \"Hydrotel\",\n",
-    "    \"project_dir\": notebook_folder / \"_data\" / \"example_hydrotel\",\n",
+    "    \"project_dir\": Path(notebook_folder) / \"_data\" / \"example_hydrotel\",\n",
     "    \"project_file\": \"projet.csv\",\n",
     "    \"simulation_config\": {\n",
     "        \"DATE DEBUT\": \"1981-01-01\",\n",

diff --git a/environment-dev.yml b/environment-dev.yml
@@ -53,7 +53,8 @@ dependencies:
   - pre-commit >=3.5.0
   - pytest >=8.3.2
   - pytest-cov >=5.0.0
-  - black  # ==24.8.0 # Waiting on https://github.com/conda-forge/black-feedstock/pull/89
+  - pytest-xdist >=3.2
+  - black ==24.8.0
   - ruff >=0.5.7
   - watchdog >=4.0.0
   - sphinx >=7.0.0

diff --git a/pyproject.toml b/pyproject.toml
@@ -81,6 +81,7 @@ dev = [
   "numpydoc >=1.8.0",
   "pytest >=8.3.2",
   "pytest-cov >=5.0.0",
+  "pytest-xdist[psutil] >=3.2",
   "black ==24.8.0",
   "blackdoc ==0.3.9",
   "isort ==5.13.2",
@@ -256,11 +257,17 @@ override_SS05 = [
 [tool.pytest.ini_options]
 addopts = [
   "--color=yes",
+  "--numprocesses=0",
+  "--maxprocesses=8",
+  "--dist=worksteal",
+  "--strict-config",
+  "--strict-markers",
   "--verbose"
 ]
 filterwarnings = ["ignore::UserWarning"]
 testpaths = "tests"
 markers = [
+  "online: mark tests that require access to the internet",
   "requires_docs: mark tests that can only be run with documentation present (deselect with '-m \"not requires_docs\"')"
 ]