diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eacb69d..848d038e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,14 +45,15 @@ jobs: create-args: >- python=${{ matrix.python-version }} attrs=22.2.0 + aioitertools=0.11.0 - name: Test with pytest - run: pytest -n auto -v -W "ignore:Downloading test file:UserWarning::" + run: pytest -n auto -v -rsx -W "ignore:Downloading test file:UserWarning::" - name: Test with pytspack installed run: | pip install https://github.com/noaa-oar-arl/pytspack/archive/master.zip - pytest -n auto -v -k with_pytspack + pytest -n auto -v -rsx -k with_pytspack docs: name: Check docs build diff --git a/monetio/sat/_omps_nadir_mm.py b/monetio/sat/_omps_nadir_mm.py index 0fe07f73..9b35dfbb 100644 --- a/monetio/sat/_omps_nadir_mm.py +++ b/monetio/sat/_omps_nadir_mm.py @@ -13,30 +13,35 @@ def read_OMPS_nm(files): import xarray as xr count = 0 - print(files) # Check if files are url if "https" in files[0]: filelist = sorted(files) for filename in filelist: data = extract_OMPS_nm_opendap(filename) - # print(data) if count == 0: data_array = data count += 1 else: data_array = xr.concat([data_array, data], "x") - else: - filelist = sorted(glob(files, recursive=False)) - - for filename in filelist: - data = extract_OMPS_nm(filename) - - if count == 0: - data_array = data - count += 1 - else: - data_array = xr.concat([data_array, data], "x") - + else: # using local files + if isinstance(files, str): # expansion of filestring to list + filelist = sorted(glob(files, recursive=False)) + else: # ensure provided filelist is sorted + filelist = sorted(files) # assume list + for filename in filelist: # extract data + try: + data = extract_OMPS_nm(filename) + if count == 0: + data_array = data + count += 1 + else: + data_array = xr.concat([data_array, data], "x") + except (KeyError, ValueError) as e: + # KeyError occurs in load when file exists but contains no data + # ValueError occurs in concat when file cross-track dimensions are different than other files loaded + print(f"warning: skipping {filename}. {type(e).__name__} occurred: {e}") + if count == 0: + raise RuntimeError(f"no files loaded from files={files}") return data_array diff --git a/tests/test_ish.py b/tests/test_ish.py index 4461cd82..80984fbd 100644 --- a/tests/test_ish.py +++ b/tests/test_ish.py @@ -6,6 +6,13 @@ from monetio import ish +try: + import requests + + requests.head("https://www1.ncdc.noaa.gov/pub/data/noaa/") +except Exception: + pytest.skip("NCEI server issues", allow_module_level=True) + def test_ish_read_history(): dates = pd.date_range("2020-09-01", "2020-09-02") diff --git a/tests/test_ish_lite.py b/tests/test_ish_lite.py index 94e9e75f..0b110ddd 100644 --- a/tests/test_ish_lite.py +++ b/tests/test_ish_lite.py @@ -3,6 +3,13 @@ from monetio import ish_lite +try: + import requests + + r = requests.head("https://www1.ncdc.noaa.gov/pub/data/noaa/isd-lite/") +except Exception: + pytest.skip("NCEI server issues", allow_module_level=True) + def test_ish_read_history(): dates = pd.date_range("2020-09-01", "2020-09-02")