From 81ea7b8838aa176bdefaf2eda7409253fdf56df9 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Thu, 3 Oct 2024 21:23:56 +0000 Subject: [PATCH 01/15] small fixes to OMPS O3 level 2 reader (_omps_nadir_mm.py) --- monetio/sat/_omps_nadir_mm.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 0fe07f73..ef3ef9a8 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -13,7 +13,7 @@ def read_OMPS_nm(files): import xarray as xr count = 0 - print(files) + #print(files) # Check if files are url if "https" in files[0]: filelist = sorted(files) @@ -26,17 +26,21 @@ def read_OMPS_nm(files): else: data_array = xr.concat([data_array, data], "x") else: - filelist = sorted(glob(files, recursive=False)) - + if isinstance(files,str): + filelist = sorted(glob(files, recursive=False)) + else: + filelist = files # assume list for filename in filelist: - data = extract_OMPS_nm(filename) - - if count == 0: - data_array = data - count += 1 - else: - data_array = xr.concat([data_array, data], "x") - + try: + data = extract_OMPS_nm(filename) + + if count == 0: + data_array = data + count += 1 + else: + data_array = xr.concat([data_array, data], "x") + except KeyError: + print(f'KeyError occured for: {filename}') return data_array From d6d4e49e9e748bf58ebc6247083a11889a9f7675 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Thu, 3 Oct 2024 22:55:29 +0000 Subject: [PATCH 02/15] corrected error handling in OMPS O3 level 2 reader --- monetio/sat/_omps_nadir_mm.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index ef3ef9a8..0397e8ac 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -29,18 +29,23 @@ def read_OMPS_nm(files): if isinstance(files,str): filelist = sorted(glob(files, recursive=False)) else: - filelist = files # assume list + filelist = sorted(files) # assume list for filename in filelist: try: data = extract_OMPS_nm(filename) - + #print(data.dims) if count == 0: data_array = data count += 1 else: data_array = xr.concat([data_array, data], "x") - except KeyError: + except KeyError as e: print(f'KeyError occured for: {filename}') + print(e) + except ValueError as e: + print(f'ValueError occured for: {filename}') + print(e) + #print(data_array.dims) return data_array @@ -158,4 +163,5 @@ def extract_OMPS_nm(fname): }, attrs={"missing_value": -999}, ) + print(ds['time'][0],ds.dims) return ds From c73bf14c6654bcccbf9e4624136f43d6fa8030e0 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Fri, 4 Oct 2024 09:15:13 -0600 Subject: [PATCH 03/15] remove debugging prints from omps_nadir --- monetio/sat/_omps_nadir_mm.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 0397e8ac..d591466b 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -19,7 +19,6 @@ def read_OMPS_nm(files): filelist = sorted(files) for filename in filelist: data = extract_OMPS_nm_opendap(filename) - # print(data) if count == 0: data_array = data count += 1 @@ -33,7 +32,6 @@ def read_OMPS_nm(files): for filename in filelist: try: data = extract_OMPS_nm(filename) - #print(data.dims) if count == 0: data_array = data count += 1 @@ -45,7 +43,6 @@ def read_OMPS_nm(files): except ValueError as e: print(f'ValueError occured for: {filename}') print(e) - #print(data_array.dims) return data_array @@ -163,5 +160,4 @@ def extract_OMPS_nm(fname): }, attrs={"missing_value": -999}, ) - print(ds['time'][0],ds.dims) return ds From 938e9ea6643f73f70f1243119dae53edf317e071 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Fri, 4 Oct 2024 13:13:59 -0600 Subject: [PATCH 04/15] remove an unnecessary print Co-authored-by: Zachary Moon --- monetio/sat/_omps_nadir_mm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index d591466b..508260f7 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -13,7 +13,6 @@ def read_OMPS_nm(files): import xarray as xr count = 0 - #print(files) # Check if files are url if "https" in files[0]: filelist = sorted(files) From 3eff6d6097c0e5a7ac13e067054c8dead1e394b8 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Fri, 4 Oct 2024 14:07:18 -0600 Subject: [PATCH 05/15] condense OMPS-NM level 2 error statement Condense exceptions and associated error statements, improved information in error statement Co-authored-by: Zachary Moon --- monetio/sat/_omps_nadir_mm.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 508260f7..3c4bf3da 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -36,12 +36,8 @@ def read_OMPS_nm(files): count += 1 else: data_array = xr.concat([data_array, data], "x") - except KeyError as e: - print(f'KeyError occured for: {filename}') - print(e) - except ValueError as e: - print(f'ValueError occured for: {filename}') - print(e) + except (KeyError, ValueError) as e: + print(f"warning: skipping {filename}. {type(e).__name__} occured: {e}") return data_array From dab10984292ea94e1abec0a8d9bb8be7f96cf82e Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Fri, 4 Oct 2024 14:21:45 -0600 Subject: [PATCH 06/15] code comments added to _omps_nadir_mm.py --- monetio/sat/_omps_nadir_mm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 3c4bf3da..2e867e1f 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -23,12 +23,12 @@ def read_OMPS_nm(files): count += 1 else: data_array = xr.concat([data_array, data], "x") - else: - if isinstance(files,str): + else: # using local files + if isinstance(files,str): # expansion of filestring to list filelist = sorted(glob(files, recursive=False)) - else: + else: # ensure provided filelist is sorted filelist = sorted(files) # assume list - for filename in filelist: + for filename in filelist: #extract data try: data = extract_OMPS_nm(filename) if count == 0: From 3188f6a712e9c557b1b1a9e2a65f72a2200012aa Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Fri, 4 Oct 2024 14:24:24 -0600 Subject: [PATCH 07/15] Add error message when no files were loaded Co-authored-by: Zachary Moon --- monetio/sat/_omps_nadir_mm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 2e867e1f..6277d385 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -38,6 +38,8 @@ def read_OMPS_nm(files): data_array = xr.concat([data_array, data], "x") except (KeyError, ValueError) as e: print(f"warning: skipping {filename}. {type(e).__name__} occured: {e}") + if count == 0: + raise RuntimeError(f"no files loaded from files={files}") return data_array From 8c027520e7ec61f111aa524594abc129a0cd4401 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Mon, 7 Oct 2024 13:56:35 -0600 Subject: [PATCH 08/15] Add KeyError/ValueError situation comments --- monetio/sat/_omps_nadir_mm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 6277d385..35e7c917 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -37,6 +37,8 @@ def read_OMPS_nm(files): else: data_array = xr.concat([data_array, data], "x") except (KeyError, ValueError) as e: + # KeyError occurs when file exists but contains no data + # ValueError occurs when file cross-track dimensions are different than other files loaded print(f"warning: skipping {filename}. {type(e).__name__} occured: {e}") if count == 0: raise RuntimeError(f"no files loaded from files={files}") From e629192c696a293946cd927d5eb18ba4c3ed2d5f Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Mon, 7 Oct 2024 14:14:26 -0600 Subject: [PATCH 09/15] Update error comments Co-authored-by: Zachary Moon --- monetio/sat/_omps_nadir_mm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 35e7c917..79f1cdf8 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -37,8 +37,8 @@ def read_OMPS_nm(files): else: data_array = xr.concat([data_array, data], "x") except (KeyError, ValueError) as e: - # KeyError occurs when file exists but contains no data - # ValueError occurs when file cross-track dimensions are different than other files loaded + # KeyError occurs in load when file exists but contains no data + # ValueError occurs in concat when file cross-track dimensions are different than other files loaded print(f"warning: skipping {filename}. {type(e).__name__} occured: {e}") if count == 0: raise RuntimeError(f"no files loaded from files={files}") From fab232061d5f3b444668212b4a39b61313c02a3a Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 7 Oct 2024 15:20:59 -0500 Subject: [PATCH 10/15] sp oops sorry that was me.. --- monetio/sat/_omps_nadir_mm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 79f1cdf8..f9575382 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -39,7 +39,7 @@ def read_OMPS_nm(files): except (KeyError, ValueError) as e: # KeyError occurs in load when file exists but contains no data # ValueError occurs in concat when file cross-track dimensions are different than other files loaded - print(f"warning: skipping {filename}. {type(e).__name__} occured: {e}") + print(f"warning: skipping {filename}. {type(e).__name__} occurred: {e}") if count == 0: raise RuntimeError(f"no files loaded from files={files}") return data_array From a3dfe78cea341ec975175bf45f9c474d487750eb Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 7 Oct 2024 15:21:40 -0500 Subject: [PATCH 11/15] fmt --- monetio/sat/_omps_nadir_mm.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index f9575382..9b35dfbb 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -23,12 +23,12 @@ def read_OMPS_nm(files): count += 1 else: data_array = xr.concat([data_array, data], "x") - else: # using local files - if isinstance(files,str): # expansion of filestring to list + else: # using local files + if isinstance(files, str): # expansion of filestring to list filelist = sorted(glob(files, recursive=False)) - else: # ensure provided filelist is sorted - filelist = sorted(files) # assume list - for filename in filelist: #extract data + else: # ensure provided filelist is sorted + filelist = sorted(files) # assume list + for filename in filelist: # extract data try: data = extract_OMPS_nm(filename) if count == 0: From 4d32de6fbfeeca8d22fe41619c1fc49528fc9851 Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 7 Oct 2024 15:40:27 -0500 Subject: [PATCH 12/15] Skip ISH if root pages unreachable --- tests/test_ish.py | 7 +++++++ tests/test_ish_lite.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/test_ish.py b/tests/test_ish.py index 4461cd82..80984fbd 100644 --- a/tests/test_ish.py +++ b/tests/test_ish.py @@ -6,6 +6,13 @@ from monetio import ish +try: + import requests + + requests.head("https://www1.ncdc.noaa.gov/pub/data/noaa/") +except Exception: + pytest.skip("NCEI server issues", allow_module_level=True) + def test_ish_read_history(): dates = pd.date_range("2020-09-01", "2020-09-02") diff --git a/tests/test_ish_lite.py b/tests/test_ish_lite.py index 94e9e75f..31aa4a24 100644 --- a/tests/test_ish_lite.py +++ b/tests/test_ish_lite.py @@ -3,6 +3,14 @@ from monetio import ish_lite +try: + import requests + + r = requests.head("https://www1.ncdc.noaa.gov/pub/data/noaa/isd-lite/") + r.raise_for_status() +except Exception: + pytest.skip("NCEI server issues", allow_module_level=True) + def test_ish_read_history(): dates = pd.date_range("2020-09-01", "2020-09-02") From 620096c8b09b4934d0e8d94c5cb11e5334507857 Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 7 Oct 2024 16:00:40 -0500 Subject: [PATCH 13/15] aioitertools 0.12.0 dropped 3.6 support https://github.com/omnilib/aioitertools/blob/main/CHANGELOG.md#v0120 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eacb69d..bb3f69ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,7 @@ jobs: create-args: >- python=${{ matrix.python-version }} attrs=22.2.0 + aioitertools=0.11.0 - name: Test with pytest run: pytest -n auto -v -W "ignore:Downloading test file:UserWarning::" From ff75d0bb342f8799aacfa3dfe6b12ab8f2191432 Mon Sep 17 00:00:00 2001 From: Zachary Moon Date: Mon, 7 Oct 2024 17:22:24 -0400 Subject: [PATCH 14/15] Don't need raise-for-status --- tests/test_ish_lite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_ish_lite.py b/tests/test_ish_lite.py index 31aa4a24..0b110ddd 100644 --- a/tests/test_ish_lite.py +++ b/tests/test_ish_lite.py @@ -7,7 +7,6 @@ import requests r = requests.head("https://www1.ncdc.noaa.gov/pub/data/noaa/isd-lite/") - r.raise_for_status() except Exception: pytest.skip("NCEI server issues", allow_module_level=True) From 03f23a14feb43d2cf2b36139569fb7591db47ba6 Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 7 Oct 2024 16:30:49 -0500 Subject: [PATCH 15/15] Show info about skipped and xfailed tests --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb3f69ee..848d038e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,12 +48,12 @@ jobs: aioitertools=0.11.0 - name: Test with pytest - run: pytest -n auto -v -W "ignore:Downloading test file:UserWarning::" + run: pytest -n auto -v -rsx -W "ignore:Downloading test file:UserWarning::" - name: Test with pytspack installed run: | pip install https://github.com/noaa-oar-arl/pytspack/archive/master.zip - pytest -n auto -v -k with_pytspack + pytest -n auto -v -rsx -k with_pytspack docs: name: Check docs build