From 3a5bd2ac25ea65ec7ac263807589513ef5f8be75 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 3 Sep 2024 11:45:24 -0600 Subject: [PATCH 1/7] fix fp tests --- .github/workflows/python-app.yml | 2 +- .../src/forcingprocessor/plot_forcings.py | 102 ++++++++------- .../src/forcingprocessor/processor.py | 4 + .../tests/test_forcingprocessor.py | 35 +++--- forcingprocessor/tests/test_fp.sh | 11 ++ forcingprocessor/tests/test_plotter.py | 116 ++++++++++++++++++ 6 files changed, 209 insertions(+), 61 deletions(-) create mode 100755 forcingprocessor/tests/test_fp.sh create mode 100644 forcingprocessor/tests/test_plotter.py diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 3af41aa5..00a02906 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -37,4 +37,4 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - pytest + ./forcingprocessor/tests/test_fp.sh diff --git a/forcingprocessor/src/forcingprocessor/plot_forcings.py b/forcingprocessor/src/forcingprocessor/plot_forcings.py index 885dca66..198b7480 100644 --- a/forcingprocessor/src/forcingprocessor/plot_forcings.py +++ b/forcingprocessor/src/forcingprocessor/plot_forcings.py @@ -15,13 +15,13 @@ mpl.use('Agg') def plot_ngen_forcings( - nwm_data : np.ndarray, - ngen_data : np.ndarray, - geopackage : str, - t_ax : list, - catchment_ids : list, - ngen_vars_plot : list, - output_dir : Path + nwm_data : np.ndarray, + ngen_data : np.ndarray, + geopackage : str, + t_ax : list, + catchment_ids : list, + ngen_vars_plot : list = ngen_variables, + output_dir : Path = './GIFs' ): """ Generates side-by-side gif of nwm and ngen forcing data @@ -42,12 +42,12 @@ def plot_ngen_forcings( print(f'creating gif for variables {nwm_variable} -> {ngen_variable}') images = [] for j, jtime in enumerate(t_ax): - fig, axes = plt.subplots(1, 2, figsize=(8, 8), dpi=200) - nwm_var = nwm_data[j,var_idx,:,:] + _, axes = plt.subplots(1, 2, figsize=(8, 8), dpi=200) + nwm_data_jvar = nwm_data[j,var_idx,:,:] if j==0: - cmin=np.min(nwm_var) - cmax=np.max(nwm_var) - im = axes[0].imshow(nwm_var, vmin=cmin, vmax=cmax) + cmin=np.min(nwm_data_jvar) + cmax=np.max(nwm_data_jvar) + im = axes[0].imshow(nwm_data_jvar, vmin=cmin, vmax=cmax) axes[0].axis('off') axes[0].set_title(f'NWM') gdf[ngen_variable] = ngen_data[j, var_idx, :] @@ -77,7 +77,9 @@ def plot_ngen_forcings( os.remove(jpng) imageio.mimsave(os.path.join(output_dir, f'{nwm_variable}_2_{ngen_variable}.gif') , images, loop=0, fps=2) -def nc_to_3darray(forcings_nc,requested_vars): +def nc_to_3darray(forcings_nc : os.PathLike, + requested_vars : list = ngen_variables + ) -> np.ndarray: ''' forcings_nc : path to ngen forcings netcdf ''' @@ -94,30 +96,60 @@ def nc_to_3darray(forcings_nc,requested_vars): return ngen_data, t_ax_dt, catchment_ids -def csvs_to_3darray(forcings_dir, requested_ngen_variables): +def csvs_to_3darray(forcings_dir : os.PathLike, + requested_vars : list = ngen_variables + ) -> np.ndarray: ''' forcings_dir : directory containing ngen forcings csvs ''' catchment_ids = [] - for (path, _, files) in os.walk(forcings_dir): + for (_, _, files) in os.walk(forcings_dir): for j, jfile in enumerate(files): - catchment_id = jfile.split('.')[0] - catchment_ids.append(catchment_id) - ngen_jdf = pd.read_csv(os.path.join(forcings_dir, jfile)) - if j == 0: - t_ax = ngen_jdf['time'] - ngen_jdf = ngen_jdf.drop(columns='time') - shp = ngen_jdf.shape - ngen_data = np.zeros((len(files),shp[0],shp[1]),dtype=np.float32) - else: - ngen_jdf = ngen_jdf.drop(columns='time') - ngen_data[j,:,:] = np.array(ngen_jdf) - - ngen_vars = np.array([x for x in range(len(ngen_variables)) if ngen_variables[x] in requested_ngen_variables]) + if jfile[-3:] == "csv": + catchment_id = jfile.split('.')[0] + catchment_ids.append(catchment_id) + ngen_jdf = pd.read_csv(os.path.join(forcings_dir, jfile)) + if j == 0: + t_ax = ngen_jdf['time'] + ngen_jdf = ngen_jdf.drop(columns='time') + shp = ngen_jdf.shape + ngen_data = np.zeros((len(files),shp[0],shp[1]),dtype=np.float32) + else: + ngen_jdf = ngen_jdf.drop(columns='time') + + ngen_data[j,:,:] = np.array(ngen_jdf) + + ngen_vars = np.array([x for x in range(len(ngen_variables)) if ngen_variables[x] in requested_vars]) ngen_data = np.moveaxis(ngen_data[:,:,ngen_vars],[0,1,2],[2,0,1]) return ngen_data, t_ax, catchment_ids +def get_nwm_data_array( + nwm_folder : list, + geopackage : gpd.GeoDataFrame, + nwm_vars : list = nwm_variables + ) -> np.ndarray: + """ + Inputs a folder of national water model files and nwm variable names to extract. + + Outputs a windowed array of national water model data for the domain and forcing variables specified. + nwm_data : 4d array (time x nwm_forcing_variable x west_east x south_north) + """ + weights_json, _ = gpkgs2weightsjson([geopackage]) + x_min, x_max, y_min, y_max = get_window(weights_json) + + for path, _, files in os.walk(nwm_folder): + nwm_data = np.zeros((len(files),len(nwm_vars),y_max-y_min+1,x_max - x_min+1),dtype=np.float32) + for k, jfile in enumerate(sorted(files)): + jfile_path = os.path.join(path,jfile) + ds = xr.open_dataset(jfile_path) + nwm_var = np.zeros((len(nwm_vars),y_max-y_min+1,x_max - x_min+1),dtype=np.float32) + for j, jvar in enumerate(nwm_vars): + nwm_var[j,:,:] = np.flip(np.squeeze(ds[jvar].isel(x=slice(x_min, x_max + 1), y=slice(3840 - y_max, 3840 - y_min + 1))),0) + nwm_data[k,:,:,:] = nwm_var + + return nwm_data + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--ngen_forcings", help="Path to a folder containing ngen catchment forcings csvs or path to netcdf",default="") @@ -127,21 +159,9 @@ def csvs_to_3darray(forcings_dir, requested_ngen_variables): parser.add_argument("--output_dir", help="Path to write gifs to",default="./GIFs") args = parser.parse_args() - weights_json, jcatchment_dict = gpkgs2weightsjson([args.geopackage]) - x_min, x_max, y_min, y_max = get_window(weights_json) - requested_ngen_variables = args.ngen_variables.split(', ') nwm_vars = np.array([nwm_variables[x] for x in range(len(ngen_variables)) if ngen_variables[x] in requested_ngen_variables]) - - for path, _, files in os.walk(args.nwm_folder): - nwm_data = np.zeros((len(files),len(nwm_vars),y_max-y_min+1,x_max - x_min+1),dtype=np.float32) - for k, jfile in enumerate(sorted(files)): - jfile_path = os.path.join(path,jfile) - ds = xr.open_dataset(jfile_path) - nwm_var = np.zeros((len(nwm_vars),y_max-y_min+1,x_max - x_min+1),dtype=np.float32) - for j, jvar in enumerate(nwm_vars): - nwm_var[j,:,:] = np.flip(np.squeeze(ds[jvar].isel(x=slice(x_min, x_max + 1), y=slice(3840 - y_max, 3840 - y_min + 1))),0) - nwm_data[k,:,:,:] = nwm_var + nwm_data = get_nwm_data_array(args.nwm_folder,args.geopackge, nwm_vars) if args.ngen_forcings.endswith('.nc'): ngen_data, t_ax, catchment_ids = nc_to_3darray(args.ngen_forcings, requested_ngen_variables) diff --git a/forcingprocessor/src/forcingprocessor/processor.py b/forcingprocessor/src/forcingprocessor/processor.py index 39cf0c00..a2ac2200 100644 --- a/forcingprocessor/src/forcingprocessor/processor.py +++ b/forcingprocessor/src/forcingprocessor/processor.py @@ -835,6 +835,10 @@ def prep_ngen_data(conf): jnwm_files = nwm_forcing_files[start:end] t0 = time.perf_counter() if ii_verbose: print(f'Entering data extraction...\n',flush=True) + # data_array, t_ax, nwm_data = forcing_grid2catchment(jnwm_files, fs) + # data_array=data_array[0][None,:] + # t_ax = t_ax + # nwm_data=nwm_data[0][None,:] data_array, t_ax, nwm_data = multiprocess_data_extract(jnwm_files,nprocs,weights_json,fs) t_extract = time.perf_counter() - t0 complexity = (nfiles_tot * ncatchments) / 10000 diff --git a/forcingprocessor/tests/test_forcingprocessor.py b/forcingprocessor/tests/test_forcingprocessor.py index f7cdc768..08c884aa 100644 --- a/forcingprocessor/tests/test_forcingprocessor.py +++ b/forcingprocessor/tests/test_forcingprocessor.py @@ -13,6 +13,8 @@ pwd = Path.cwd() pwd = pwd data_dir = data_dir +if os.path.exists(data_dir): + os.system(f"rm -rf {data_dir}") os.system(f"mkdir {data_dir}") pwd = Path.cwd() filenamelist = str((pwd/"filenamelist.txt").resolve()) @@ -34,7 +36,8 @@ "run" : { "verbose" : False, - "collect_stats" : True + "collect_stats" : True, + "nprocs" : 1 } } @@ -72,9 +75,7 @@ def test_nomads_prod(): os.remove(parquet) def test_nomads_post_processed(): - print(f'test_nomads_post_processed() is BROKEN - https://github.com/CIROH-UA/nwmurl/issues/62') - assert False - return + assert False, f'test_nomads_post_processed() is BROKEN - https://github.com/CIROH-UA/nwmurl/issues/62' nwmurl_conf['start_date'] = "202408240000" nwmurl_conf['end_date'] = "202408241700" nwmurl_conf["urlbaseinput"] = 2 @@ -92,11 +93,9 @@ def test_nwm_google_apis(): prep_ngen_data(conf) parquet = (data_dir/"forcings/cat-2586011.parquet").resolve() assert parquet.exists() - os.remove(parquet) + os.remove(parquet) def test_google_cloud_storage(): - print(f'hangs in pytest, but should work') - return nwmurl_conf['start_date'] = "202407100100" nwmurl_conf['end_date'] = "202407100100" nwmurl_conf["urlbaseinput"] = 4 @@ -104,11 +103,9 @@ def test_google_cloud_storage(): prep_ngen_data(conf) parquet = (data_dir/"forcings/cat-2586011.parquet").resolve() assert parquet.exists() - os.remove(parquet) + os.remove(parquet) -def test_gs_nwm(): - print(f'hangs in pytest, but should work') - return +def test_gs(): nwmurl_conf['start_date'] = date + hourminute nwmurl_conf['end_date'] = date + hourminute nwmurl_conf["urlbaseinput"] = 5 @@ -116,19 +113,18 @@ def test_gs_nwm(): prep_ngen_data(conf) parquet = (data_dir/"forcings/cat-2586011.parquet").resolve() assert parquet.exists() - os.remove(parquet) + os.remove(parquet) -def test_gcs_nwm(): - print(f'hangs in pytest, but should work') - return - nwmurl_conf['start_date'] = date + hourminute - nwmurl_conf['end_date'] = date + hourminute +def test_gcs(): + # assert False, f'hangs in pytest, but should work' + nwmurl_conf['start_date'] = "202407100100" + nwmurl_conf['end_date'] = "202407100100" nwmurl_conf["urlbaseinput"] = 6 generate_nwmfiles(nwmurl_conf) prep_ngen_data(conf) parquet = (data_dir/"forcings/cat-2586011.parquet").resolve() assert parquet.exists() - os.remove(parquet) + os.remove(parquet) def test_noaa_nwm_pds(): nwmurl_conf['start_date'] = date + hourminute @@ -200,6 +196,7 @@ def test_retro_3_0(): def test_plotting(): conf['forcing']['nwm_file'] = retro_filenamelist + conf['plot'] = {} conf['plot']['nts'] = 1 conf['plot']['ngen_vars'] = [ "TMP_2maboveground" @@ -207,7 +204,7 @@ def test_plotting(): nwmurl_conf_retro["urlbaseinput"] = 4 generate_nwmfiles(nwmurl_conf_retro) prep_ngen_data(conf) - GIF = (data_dir/"forcings/metadata/T2D_2_TMP_2maboveground.gif").resolve() + GIF = (data_dir/"metadata/GIFs/T2D_2_TMP_2maboveground.gif").resolve() assert GIF.exists() os.remove(GIF) diff --git a/forcingprocessor/tests/test_fp.sh b/forcingprocessor/tests/test_fp.sh new file mode 100755 index 00000000..6eb786b3 --- /dev/null +++ b/forcingprocessor/tests/test_fp.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +google_tests=$(sort "forcingprocessor/tests/google.txt") +broken_tests=$(sort "forcingprocessor/tests/broken.txt") +deselect_tests=$(sort "forcingprocessor/tests/google.txt" "forcingprocessor/tests/broken.txt") +echo $deselect_tests + +python -m pytest -vv --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_google_cloud_storage" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_gcs" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_gs" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_ciroh_zarr" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_nomads_post_processed" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_retro_ciroh_zarr" +python -m pytest -v -k test_google_cloud_storage +python -m pytest -v -k test_gs +python -m pytest -v -k test_gcs \ No newline at end of file diff --git a/forcingprocessor/tests/test_plotter.py b/forcingprocessor/tests/test_plotter.py new file mode 100644 index 00000000..d8032fd0 --- /dev/null +++ b/forcingprocessor/tests/test_plotter.py @@ -0,0 +1,116 @@ +import os +from pathlib import Path +from datetime import datetime +from datetime import datetime +from forcingprocessor.processor import prep_ngen_data +from forcingprocessor.plot_forcings import nc_to_3darray, plot_ngen_forcings, csvs_to_3darray, get_nwm_data_array +from forcingprocessor.nwm_filenames_generator import generate_nwmfiles +import requests + +date = datetime.now() +date = date.strftime('%Y%m%d') +hourminute = '0000' +test_dir = Path(__file__).parent +data_dir = (test_dir/'data').resolve() +if os.path.exists(data_dir): + os.system(f"rm -rf {data_dir}") +os.system(f"mkdir {data_dir}") +pwd = Path.cwd() +filenamelist = str((pwd/"filenamelist.txt").resolve()) +geopackage = str(f"{data_dir}/palisade.gpkg") +geopackage_name = "palisade.gpkg" +os.system(f"curl -o {os.path.join(data_dir,geopackage_name)} -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/{geopackage_name}") + +conf = { + "forcing" : { + "nwm_file" : filenamelist, + "gpkg_file" : geopackage + }, + + "storage":{ + "storage_type" : "local", + "output_path" : str(data_dir), + "output_file_type" : ["netcdf","csv"] + }, + + "run" : { + "verbose" : False, + "collect_stats" : True, + "nproc" : 8 + } + } + +nwmurl_conf_retro = { + "forcing_type" : "retrospective", + "start_date" : "201801010000", + "end_date" : "201801012300", + "urlbaseinput" : 1, + "selected_object_type" : [1], + "selected_var_types" : [6], + "write_to_file" : True + } + +nwmurl_conf = { + "forcing_type" : "operational_archive", + "start_date" : "", + "end_date" : "", + "runinput" : 1, + "varinput" : 5, + "geoinput" : 1, + "meminput" : 0, + "urlbaseinput" : 7, + "fcst_cycle" : [0], + "lead_time" : [1] + } + +def test_forcings_plot(): + os.system(f"mkdir {data_dir}") + nwmurl_conf['start_date'] = date + hourminute + nwmurl_conf['end_date'] = date + hourminute + nwmurl_conf["urlbaseinput"] = 7 + generate_nwmfiles(nwmurl_conf) + prep_ngen_data(conf) + + nwm_dir = os.path.join(data_dir,'nwm_forcings') + if not os.path.exists(nwm_dir): + os.system(f"mkdir {nwm_dir}") + + filenamelist_file = "./filenamelist.txt" + with open(filenamelist_file,'r') as fp: + urls = fp.readlines() + for jurl in urls: + response = requests.get(jurl.strip()) + file_Path = os.path.join(nwm_dir,os.path.basename(jurl.strip())) + if response.status_code == 200: + with open(file_Path, 'wb') as file: + file.write(response.content) + print('File downloaded successfully') + else: + raise Exception(f'Failed to download file') + + nwm_data = get_nwm_data_array(nwm_dir,geopackage) + + forcings_nc = os.path.join(data_dir,"forcings/1_forcings.nc") + ngen_data, t_ax, catchment_ids = nc_to_3darray(forcings_nc) + + plot_ngen_forcings( + nwm_data, + ngen_data, + geopackage, + t_ax, + catchment_ids, + ["TMP_2maboveground"], + os.path.join(data_dir,'metadata/GIFs') + ) + + os.system(f'rm {forcings_nc}') + ngen_data, t_ax, catchment_ids = csvs_to_3darray(os.path.join(data_dir,'forcings')) + plot_ngen_forcings( + nwm_data, + ngen_data, + geopackage, + t_ax, + catchment_ids, + ["TMP_2maboveground"], + os.path.join(data_dir,'metadata/GIFs') + ) \ No newline at end of file From b011cd3b7b440fce55030838a7c2e27415ba80d8 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 3 Sep 2024 11:48:44 -0600 Subject: [PATCH 2/7] Update python-app.yml --- .github/workflows/python-app.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 00a02906..b611d4bc 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -29,12 +29,6 @@ jobs: pip install -e ./forcingprocessor pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | ./forcingprocessor/tests/test_fp.sh From 2eb5aa63271b0231aa0825237785d6f68ca13f06 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 3 Sep 2024 11:51:47 -0600 Subject: [PATCH 3/7] remove text files --- forcingprocessor/tests/test_fp.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/forcingprocessor/tests/test_fp.sh b/forcingprocessor/tests/test_fp.sh index 6eb786b3..6197062a 100755 --- a/forcingprocessor/tests/test_fp.sh +++ b/forcingprocessor/tests/test_fp.sh @@ -1,10 +1,5 @@ #!/bin/bash -google_tests=$(sort "forcingprocessor/tests/google.txt") -broken_tests=$(sort "forcingprocessor/tests/broken.txt") -deselect_tests=$(sort "forcingprocessor/tests/google.txt" "forcingprocessor/tests/broken.txt") -echo $deselect_tests - python -m pytest -vv --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_google_cloud_storage" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_gcs" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_gs" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_ciroh_zarr" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_nomads_post_processed" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_retro_ciroh_zarr" python -m pytest -v -k test_google_cloud_storage python -m pytest -v -k test_gs From c868e4668577cfb5899a6f86c53d0c292b5baec5 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 3 Sep 2024 11:54:56 -0600 Subject: [PATCH 4/7] add imageio --- forcingprocessor/setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/forcingprocessor/setup.cfg b/forcingprocessor/setup.cfg index 6bf839c5..59fdc20e 100644 --- a/forcingprocessor/setup.cfg +++ b/forcingprocessor/setup.cfg @@ -31,6 +31,7 @@ install_requires = cftime gcsfs geopandas + imageio matplotlib netCDF4 h5netcdf From 9c231b1eec68fdd2065804909b01e4c8cf05a267 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 3 Sep 2024 12:08:20 -0600 Subject: [PATCH 5/7] change test name --- .github/workflows/{python-app.yml => forcingprocessor.yml} | 6 ++---- forcingprocessor/src/forcingprocessor/plot_forcings.py | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) rename .github/workflows/{python-app.yml => forcingprocessor.yml} (84%) diff --git a/.github/workflows/python-app.yml b/.github/workflows/forcingprocessor.yml similarity index 84% rename from .github/workflows/python-app.yml rename to .github/workflows/forcingprocessor.yml index b611d4bc..f2b1496d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/forcingprocessor.yml @@ -1,8 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: Python application - +name: Test Forcing Processor on: push: branches: [ "main" ] @@ -27,8 +26,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ./forcingprocessor - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install pytest - name: Test with pytest run: | ./forcingprocessor/tests/test_fp.sh diff --git a/forcingprocessor/src/forcingprocessor/plot_forcings.py b/forcingprocessor/src/forcingprocessor/plot_forcings.py index 198b7480..e4a4e811 100644 --- a/forcingprocessor/src/forcingprocessor/plot_forcings.py +++ b/forcingprocessor/src/forcingprocessor/plot_forcings.py @@ -103,13 +103,15 @@ def csvs_to_3darray(forcings_dir : os.PathLike, forcings_dir : directory containing ngen forcings csvs ''' catchment_ids = [] + i = 0 for (_, _, files) in os.walk(forcings_dir): for j, jfile in enumerate(files): if jfile[-3:] == "csv": catchment_id = jfile.split('.')[0] catchment_ids.append(catchment_id) ngen_jdf = pd.read_csv(os.path.join(forcings_dir, jfile)) - if j == 0: + if i == 0: + i += 1 t_ax = ngen_jdf['time'] ngen_jdf = ngen_jdf.drop(columns='time') shp = ngen_jdf.shape From ac340f167c8b22599d1ce1b8a7ce09602360c528 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 3 Sep 2024 12:11:49 -0600 Subject: [PATCH 6/7] move shell to yml --- .github/workflows/forcingprocessor.yml | 5 ++++- forcingprocessor/tests/test_fp.sh | 6 ------ 2 files changed, 4 insertions(+), 7 deletions(-) delete mode 100755 forcingprocessor/tests/test_fp.sh diff --git a/.github/workflows/forcingprocessor.yml b/.github/workflows/forcingprocessor.yml index f2b1496d..17435ff2 100644 --- a/.github/workflows/forcingprocessor.yml +++ b/.github/workflows/forcingprocessor.yml @@ -29,4 +29,7 @@ jobs: pip install pytest - name: Test with pytest run: | - ./forcingprocessor/tests/test_fp.sh + python -m pytest -vv --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_google_cloud_storage" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_gcs" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_gs" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_ciroh_zarr" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_nomads_post_processed" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_retro_ciroh_zarr" + python -m pytest -v -k test_google_cloud_storage + python -m pytest -v -k test_gs + python -m pytest -v -k test_gcs diff --git a/forcingprocessor/tests/test_fp.sh b/forcingprocessor/tests/test_fp.sh deleted file mode 100755 index 6197062a..00000000 --- a/forcingprocessor/tests/test_fp.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -python -m pytest -vv --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_google_cloud_storage" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_gcs" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_gs" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_ciroh_zarr" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_nomads_post_processed" --deselect="forcingprocessor/tests/test_forcingprocessor.py::test_retro_ciroh_zarr" -python -m pytest -v -k test_google_cloud_storage -python -m pytest -v -k test_gs -python -m pytest -v -k test_gcs \ No newline at end of file From 21ed2209ec718711a7cc06c8f9b90bb685d587b7 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 3 Sep 2024 13:01:02 -0600 Subject: [PATCH 7/7] fix plotter --- forcingprocessor/tests/test_plotter.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/forcingprocessor/tests/test_plotter.py b/forcingprocessor/tests/test_plotter.py index d8032fd0..4befce76 100644 --- a/forcingprocessor/tests/test_plotter.py +++ b/forcingprocessor/tests/test_plotter.py @@ -12,14 +12,10 @@ hourminute = '0000' test_dir = Path(__file__).parent data_dir = (test_dir/'data').resolve() -if os.path.exists(data_dir): - os.system(f"rm -rf {data_dir}") -os.system(f"mkdir {data_dir}") pwd = Path.cwd() filenamelist = str((pwd/"filenamelist.txt").resolve()) geopackage = str(f"{data_dir}/palisade.gpkg") geopackage_name = "palisade.gpkg" -os.system(f"curl -o {os.path.join(data_dir,geopackage_name)} -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/{geopackage_name}") conf = { "forcing" : { @@ -64,10 +60,11 @@ } def test_forcings_plot(): - os.system(f"mkdir {data_dir}") nwmurl_conf['start_date'] = date + hourminute nwmurl_conf['end_date'] = date + hourminute nwmurl_conf["urlbaseinput"] = 7 + os.system(f"rm -rf {data_dir}/forcings/*.parquet") + generate_nwmfiles(nwmurl_conf) prep_ngen_data(conf)