From 8876e78f8356245fc00d086b4a5cf16a9ace7319 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Fri, 1 Mar 2024 14:03:31 -0500 Subject: [PATCH 01/11] Added refine_Atmos_no_redux.py as is from fre2/postprocessing app/ location --- fre/freapp/maskAtmosPlevel.py | 188 ++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100755 fre/freapp/maskAtmosPlevel.py diff --git a/fre/freapp/maskAtmosPlevel.py b/fre/freapp/maskAtmosPlevel.py new file mode 100755 index 00000000..709e121d --- /dev/null +++ b/fre/freapp/maskAtmosPlevel.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +# This script contains the refineDiags that produce data at the same +# frequency as the input data (no reduction) such as surface albedo, +# masking fields,... +# It can accept any file and will only compute refineDiags in fields +# are present. + +import argparse +import os +import netCDF4 as nc +import xarray as xr + +def run(): + args = parse_args() + + # Error if outfile exists + if os.path.exists(args.outfile): + raise Exception(f"ERROR: Output file '{args.outfile}' already exists") + + # Open ps dataset + ds_ps = xr.open_dataset(args.ps) + + # Exit with message if "ps" not available + if "ps" not in list(ds_ps.variables): + print(f"WARNING: File '{args.infile}' does not contain surface pressure, so exiting") + return None + + # Open input dataset + ds_in = xr.open_dataset(args.infile) + + # The trigger for atmos masking is a variable attribute "needs_atmos_masking = True". + # In the future this will be set within the model, but for now and testing, + # we'll add the attribute for variables that end with "_unmsk". + ds_in = preprocess(ds_in) + + ds_out = xr.Dataset() + + # Process all variables with attribute "needs_atmos_masking = True" + for var in list(ds_in.variables): + if 'needs_atmos_masking' in ds_in[var].attrs: + del ds_in[var].attrs['needs_atmos_masking'] + ds_out[var] = mask_field_above_surface_pressure(ds_in, var, ds_ps) + else: + continue + + # Write the output file if anything was done + if ds_out.variables: + print(f"Modifying variables '{list(ds_out.variables)}', appending into new file '{args.outfile}'") + write_dataset(ds_out, ds_in, args.outfile) + else: + print(f"No variables modified, so not writing output file '{args.outfile}'") + return None + + +def preprocess(ds): + """add needs_atmos_masking attribute if var ends with _unmsk""" + + for var in list(ds.variables): + if var.endswith('_unmsk'): + ds[var].attrs['needs_atmos_masking'] = True + + return ds + + +def mask_field_above_surface_pressure(ds, var, ds_ps): + """mask data with pressure larger than surface pressure""" + + plev = pressure_coordinate(ds, var) + + # broadcast pressure coordinate and surface pressure to + # the dimensions of the variable to mask + plev_extended, _ = xr.broadcast(plev, ds[var]) + ps_extended, _ = xr.broadcast(ds_ps["ps"], ds[var]) + # masking do not need looping + masked = xr.where(plev_extended > ps_extended, 1.0e20, ds[var]) + # copy attributes, but it doesn't include the missing values + attrs = ds[var].attrs.copy() + # add the missing values back + attrs['missing_value'] = 1.0e20 + attrs['_FillValue'] = 1.0e20 + masked.attrs = attrs + # transpose dims like the original array + masked = masked.transpose(*ds[var].dims) + + print(f"Processed {var}") + + return masked + + +def pressure_coordinate(ds, varname, verbose=False): + """check if dataArray has pressure coordinate fitting requirements + and return it""" + + pressure_coord = None + + for dim in list(ds[varname].dims): + if dim in list(ds.variables): # dim needs to have values in file + if ds[dim].attrs["long_name"] == "pressure": + pressure_coord = ds[dim] + elif ("coordinates" in ds.attrs) and (ds[dim].attrs["units"] == "Pa"): + pressure_coord = ds[dim] + + return pressure_coord + + +def write_dataset(ds, template, outfile): + """prepare the dataset and dump into netcdf file""" + + # copy global attributes + ds.attrs = template.attrs.copy() + + # copy all variables and their attributes + # except those already processed + for var in list(template.variables): + if var in list(ds.variables): + continue + else: + ds[var] = template[var] + ds[var].attrs = template[var].attrs.copy() + + ds.to_netcdf(outfile, unlimited_dims="time") + + return None + + +def set_netcdf_encoding(ds, pressure_vars): + """set preferred options for netcdf encoding""" + + all_vars = list(ds.variables) + encoding = {} + + for var in do_not_encode_vars + pressure_vars: + if var in all_vars: + encoding.update({var: dict(_FillValue=None)}) + + return encoding + + +def post_write(filename, ds, var_with_bounds, bounds_variables): + """fix a posteriori attributes that xarray.to_netcdf + did not do properly using low level netcdf lib""" + + f = nc.Dataset(filename, "a") + + for var, bndvar in zip(var_with_bounds, bounds_variables): + f.variables[var].setncattr("bounds", bndvar) + + f.close() + + return None + + +def parse_args(): + """parse command line arguments""" + + parser = argparse.ArgumentParser() + parser.add_argument("infile", type=str, help="Input file") + parser.add_argument( + "-o", "--outfile", type=str, required=True, help="Output file name" + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + required=False, + help="Print detailed output", + ) + parser.add_argument( + "-f", + "--format", + type=str, + required=False, + default="NETCDF3_64BIT", + help="netcdf format for output file", + ) + parser.add_argument( + "-p", + "--ps", + type=str, + required=True, + help="NetCDF file containing surface pressure" + ) + return parser.parse_args() + + +if __name__ == "__main__": + run() From 81335ebbf00ee511dd395da14f0e4330550fbaf6 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Fri, 1 Mar 2024 14:05:11 -0500 Subject: [PATCH 02/11] Added click scaffolding --- fre/__init__.py | 1 + fre/fre.py | 30 ++++++++++++++++++++++++++++++ fre/freapp/__init__.py | 1 + fre/freapp/freapp.py | 3 +++ 4 files changed, 35 insertions(+) create mode 100644 fre/freapp/__init__.py create mode 100644 fre/freapp/freapp.py diff --git a/fre/__init__.py b/fre/__init__.py index 903c60b1..46f50bd1 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -6,3 +6,4 @@ from fre.frerun import * from fre.frecatalog import * from fre.freyamltools import * +from fre.freapp import * diff --git a/fre/fre.py b/fre/fre.py index f6e40851..c7ba92c5 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -31,6 +31,9 @@ from fre import freyamltools from fre.freyamltools.freyamltools import * +from fre import freapp +from fre.freapp.freapp import * + ############################################# """ @@ -85,6 +88,33 @@ def freYamltools(): """ - access fre yamltools subcommands """ pass +@fre.group('app') +def freApp(): + """access fre app subcommands""" + pass + +############################################# + +""" +fre app subcommands to be processed +""" +@freApp.command() +@click.option("-i", "--infile", + type=str, + help="Input NetCDF file containing pressure-level output to be masked", + required=True) +@click.option("-o", "--outfile", + type=str, + help="Output file", + required=True) +@click.option("-p", "--psfile", + help="Input NetCDF file containing surface pressure (ps)", + required=True) +@click.pass_context +def maskAtmosPlevel(context, infile, outfile, psfile): + """Mask out pressure level diagnostic output below land surface""" + context.forward(freapp.freapp.maskAtmosPlevel_subtool) + ############################################# """ diff --git a/fre/freapp/__init__.py b/fre/freapp/__init__.py new file mode 100644 index 00000000..9edbc2d1 --- /dev/null +++ b/fre/freapp/__init__.py @@ -0,0 +1 @@ +from fre.freapp import * diff --git a/fre/freapp/freapp.py b/fre/freapp/freapp.py new file mode 100644 index 00000000..7956434b --- /dev/null +++ b/fre/freapp/freapp.py @@ -0,0 +1,3 @@ +#!/usr/bin/python3 + +from fre.freapp.maskAtmosPlevel import * From 71bd37afd6c175154323df29fbf7aafaa8d37b3d Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Fri, 1 Mar 2024 16:20:42 -0500 Subject: [PATCH 03/11] Tiny changes to maskAtmosPlevel for click callback --- fre/freapp/maskAtmosPlevel.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/fre/freapp/maskAtmosPlevel.py b/fre/freapp/maskAtmosPlevel.py index 709e121d..313606bb 100755 --- a/fre/freapp/maskAtmosPlevel.py +++ b/fre/freapp/maskAtmosPlevel.py @@ -10,24 +10,29 @@ import os import netCDF4 as nc import xarray as xr +import click -def run(): - args = parse_args() +@click.command() +def maskAtmosPlevel_subtool(infile, outfile, psfile): # Error if outfile exists - if os.path.exists(args.outfile): - raise Exception(f"ERROR: Output file '{args.outfile}' already exists") + if os.path.exists(outfile): + raise Exception(f"ERROR: Output file '{outfile}' already exists") # Open ps dataset - ds_ps = xr.open_dataset(args.ps) + if not os.path.exists(psfile): + raise Exception(f"ERROR: Input surface pressure file '{psfile}' does not exist") + ds_ps = xr.open_dataset(psfile) # Exit with message if "ps" not available if "ps" not in list(ds_ps.variables): - print(f"WARNING: File '{args.infile}' does not contain surface pressure, so exiting") + print(f"WARNING: File '{infile}' does not contain surface pressure, so exiting") return None # Open input dataset - ds_in = xr.open_dataset(args.infile) + if not os.path.exists(infile): + raise Exception(f"ERROR: Input file '{infile}' does not exist") + ds_in = xr.open_dataset(infile) # The trigger for atmos masking is a variable attribute "needs_atmos_masking = True". # In the future this will be set within the model, but for now and testing, @@ -46,10 +51,10 @@ def run(): # Write the output file if anything was done if ds_out.variables: - print(f"Modifying variables '{list(ds_out.variables)}', appending into new file '{args.outfile}'") - write_dataset(ds_out, ds_in, args.outfile) + print(f"Modifying variables '{list(ds_out.variables)}', appending into new file '{outfile}'") + write_dataset(ds_out, ds_in, outfile) else: - print(f"No variables modified, so not writing output file '{args.outfile}'") + print(f"No variables modified, so not writing output file '{outfile}'") return None @@ -185,4 +190,6 @@ def parse_args(): if __name__ == "__main__": - run() + args = parse_args() + + mask_subtool(args.infile, args.outfile, args.ps) From 86c71487fcc799a98d6e9e6615686105b7ec66e9 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Fri, 1 Mar 2024 16:28:28 -0500 Subject: [PATCH 04/11] Rename subtool to mask-atmos-plevel (with dashes) --- fre/fre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/fre.py b/fre/fre.py index c7ba92c5..76360074 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -111,7 +111,7 @@ def freApp(): help="Input NetCDF file containing surface pressure (ps)", required=True) @click.pass_context -def maskAtmosPlevel(context, infile, outfile, psfile): +def mask_atmos_plevel(context, infile, outfile, psfile): """Mask out pressure level diagnostic output below land surface""" context.forward(freapp.freapp.maskAtmosPlevel_subtool) From cfb44d15a41f3bff056b0182494fcec49ff68175 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 18 Mar 2024 16:56:03 -0400 Subject: [PATCH 05/11] Removed argparse from fre app mask-atmos-plevel as it's not needed --- fre/freapp/maskAtmosPlevel.py | 40 ----------------------------------- 1 file changed, 40 deletions(-) diff --git a/fre/freapp/maskAtmosPlevel.py b/fre/freapp/maskAtmosPlevel.py index 313606bb..bb430dc9 100755 --- a/fre/freapp/maskAtmosPlevel.py +++ b/fre/freapp/maskAtmosPlevel.py @@ -6,7 +6,6 @@ # It can accept any file and will only compute refineDiags in fields # are present. -import argparse import os import netCDF4 as nc import xarray as xr @@ -154,42 +153,3 @@ def post_write(filename, ds, var_with_bounds, bounds_variables): f.close() return None - - -def parse_args(): - """parse command line arguments""" - - parser = argparse.ArgumentParser() - parser.add_argument("infile", type=str, help="Input file") - parser.add_argument( - "-o", "--outfile", type=str, required=True, help="Output file name" - ) - parser.add_argument( - "-v", - "--verbose", - action="store_true", - required=False, - help="Print detailed output", - ) - parser.add_argument( - "-f", - "--format", - type=str, - required=False, - default="NETCDF3_64BIT", - help="netcdf format for output file", - ) - parser.add_argument( - "-p", - "--ps", - type=str, - required=True, - help="NetCDF file containing surface pressure" - ) - return parser.parse_args() - - -if __name__ == "__main__": - args = parse_args() - - mask_subtool(args.infile, args.outfile, args.ps) From 2610c5ccda0cecac2a44d1996ccd74578ce861ad Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 18 Mar 2024 20:48:50 -0400 Subject: [PATCH 06/11] Added some simple meta.yaml checks --- meta.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta.yaml b/meta.yaml index 4be9437c..4d34224e 100644 --- a/meta.yaml +++ b/meta.yaml @@ -39,6 +39,7 @@ test: - fre.frepp.status - fre.frepp.run - fre.frepp.validate + - fre.app commands: - fre --help - fre pp --help @@ -46,6 +47,8 @@ test: - fre pp status --help - fre pp run --help - fre pp validate --help + - fre app --help + - fre app mask-atmos-plevel --help about: home: https://github.com/NOAA-GFDL/fre-cli From 4695b4f43866a066a69a7d012f5f388eee47110e Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 18 Mar 2024 21:00:33 -0400 Subject: [PATCH 07/11] Corrected meta.yaml python import check. will be fre.app soon --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 4d34224e..2995cbc6 100644 --- a/meta.yaml +++ b/meta.yaml @@ -39,7 +39,7 @@ test: - fre.frepp.status - fre.frepp.run - fre.frepp.validate - - fre.app + - fre.freapp commands: - fre --help - fre pp --help From 1cf3ab8ea8e0df4fb9f8766a5946c2a8a420a4b3 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Thu, 21 Mar 2024 20:48:21 -0400 Subject: [PATCH 08/11] Update meta.yaml Comment out the import test temporarily --- meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 2995cbc6..5069c23b 100644 --- a/meta.yaml +++ b/meta.yaml @@ -39,7 +39,8 @@ test: - fre.frepp.status - fre.frepp.run - fre.frepp.validate - - fre.freapp + # next line should import but it does not due to mystery + #- fre.freapp commands: - fre --help - fre pp --help From 6a78289c0eceb0121e0d33df72cf306bb89b2a3f Mon Sep 17 00:00:00 2001 From: Bennett Chang Date: Tue, 2 Apr 2024 12:53:09 -0400 Subject: [PATCH 09/11] fre app refactor changes --- fre/__init__.py | 2 +- fre/app/__init__.py | 3 +++ fre/{freapp => app}/maskAtmosPlevel.py | 1 - fre/fre.py | 9 ++++----- fre/freapp/__init__.py | 1 - fre/freapp/freapp.py | 3 --- meta.yaml | 5 ++--- 7 files changed, 10 insertions(+), 14 deletions(-) create mode 100644 fre/app/__init__.py rename fre/{freapp => app}/maskAtmosPlevel.py (99%) delete mode 100644 fre/freapp/__init__.py delete mode 100644 fre/freapp/freapp.py diff --git a/fre/__init__.py b/fre/__init__.py index 46f50bd1..9668791a 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -6,4 +6,4 @@ from fre.frerun import * from fre.frecatalog import * from fre.freyamltools import * -from fre.freapp import * +from .app import * diff --git a/fre/app/__init__.py b/fre/app/__init__.py new file mode 100644 index 00000000..8715de49 --- /dev/null +++ b/fre/app/__init__.py @@ -0,0 +1,3 @@ +from .maskAtmosPlevel import maskAtmosPlevel_subtool + +__all__ = ["maskAtmosPlevel_subtool"] diff --git a/fre/freapp/maskAtmosPlevel.py b/fre/app/maskAtmosPlevel.py similarity index 99% rename from fre/freapp/maskAtmosPlevel.py rename to fre/app/maskAtmosPlevel.py index bb430dc9..fd6d5896 100755 --- a/fre/freapp/maskAtmosPlevel.py +++ b/fre/app/maskAtmosPlevel.py @@ -12,7 +12,6 @@ import click @click.command() - def maskAtmosPlevel_subtool(infile, outfile, psfile): # Error if outfile exists if os.path.exists(outfile): diff --git a/fre/fre.py b/fre/fre.py index 76360074..591bb832 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -31,8 +31,7 @@ from fre import freyamltools from fre.freyamltools.freyamltools import * -from fre import freapp -from fre.freapp.freapp import * +from .app import maskAtmosPlevel_subtool ############################################# @@ -89,7 +88,7 @@ def freYamltools(): pass @fre.group('app') -def freApp(): +def freapp(): """access fre app subcommands""" pass @@ -98,7 +97,7 @@ def freApp(): """ fre app subcommands to be processed """ -@freApp.command() +@freapp.command() @click.option("-i", "--infile", type=str, help="Input NetCDF file containing pressure-level output to be masked", @@ -113,7 +112,7 @@ def freApp(): @click.pass_context def mask_atmos_plevel(context, infile, outfile, psfile): """Mask out pressure level diagnostic output below land surface""" - context.forward(freapp.freapp.maskAtmosPlevel_subtool) + context.forward(maskAtmosPlevel_subtool) ############################################# diff --git a/fre/freapp/__init__.py b/fre/freapp/__init__.py deleted file mode 100644 index 9edbc2d1..00000000 --- a/fre/freapp/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from fre.freapp import * diff --git a/fre/freapp/freapp.py b/fre/freapp/freapp.py deleted file mode 100644 index 7956434b..00000000 --- a/fre/freapp/freapp.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/python3 - -from fre.freapp.maskAtmosPlevel import * diff --git a/meta.yaml b/meta.yaml index 5069c23b..3ae8da6b 100644 --- a/meta.yaml +++ b/meta.yaml @@ -10,7 +10,7 @@ source: build: script: {{ PYTHON }} -m pip install . -vv - number: 0 + number: 1 noarch: python channels: @@ -39,8 +39,7 @@ test: - fre.frepp.status - fre.frepp.run - fre.frepp.validate - # next line should import but it does not due to mystery - #- fre.freapp + - fre.freapp commands: - fre --help - fre pp --help From 5cc1e9fdf36add76fe4237843085944ae4d5ab3d Mon Sep 17 00:00:00 2001 From: Bennett Chang Date: Tue, 2 Apr 2024 13:38:54 -0400 Subject: [PATCH 10/11] meta.yaml testing change --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 3ae8da6b..ce2b192e 100644 --- a/meta.yaml +++ b/meta.yaml @@ -39,7 +39,7 @@ test: - fre.frepp.status - fre.frepp.run - fre.frepp.validate - - fre.freapp + - fre.app commands: - fre --help - fre pp --help From 9520605cbbf9713ef34ad811ac700f1024ccde71 Mon Sep 17 00:00:00 2001 From: Bennett Chang Date: Tue, 2 Apr 2024 13:46:52 -0400 Subject: [PATCH 11/11] meta.yaml removal of git url --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index ce2b192e..2835dae2 100644 --- a/meta.yaml +++ b/meta.yaml @@ -5,8 +5,8 @@ package: version: {{ data.get('version') }} source: - git_url: https://github.com/NOAA-GFDL/fre-cli path: . +# git_url: https://github.com/NOAA-GFDL/fre-cli build: script: {{ PYTHON }} -m pip install . -vv