From 689c065a5fb280203fb46aca5406e6ccbc371d5b Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 1 Nov 2024 13:42:48 -0400 Subject: [PATCH 01/36] Reapply "intermediate commit- new function to compar variable dimensions. none checks, trying to generally allow this code to be more flexible. not currently working atm" This reverts commit 1a15d9534c767ba85061d529089fdb7e06832469. --- fre/cmor/cmor_mixer.py | 195 +++++++++++++++++++++++++++++++++++------ run_test_file_cases.py | 9 +- 2 files changed, 175 insertions(+), 29 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 0f4ef243..6ba4cdf3 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -20,6 +20,61 @@ # ----- \end consts ### ------ helper functions ------ ### +def check_table_dims_v_var_dims(json_table_config = None, table_var_dims = None, var_dims = None): + ''' + checks the mip table's required dimensions against the current input netcdf file's dimensions. if a req'd + dimension isn't found in the input netcdf file, either as an exact match or a match to a standard/long/out name field, + we throw an error. currently accepts three arguments, two required and one optional + json_table_config: string, optional + table_var_dims: tuple of strings, req'd + var_dims: tuple of strings, req'd + ''' + # check if the information req'd by the table is present in the list assoc with the variable + print(f'\n\n(check_table_dims_v_var_dims) attempting to compare target_var dimension names with table variable entry dimension names...') + for proj_table_var_dim in table_var_dims: + print(f'-----------------------------------------------------------------------------------------------------------------------------------------') + print(f'(check_table_dims_v_var_dims) proj_table_var_dim = {proj_table_var_dim}') + + # check the mip coordinate file for accepted standard_name, out_name, long_name flavors + cmip_coords = None + try: + json_coordinate_config = str(Path(json_table_config).parent) + '/CMIP6_coordinate.json' + print(f'(check_table_dims_v_var_dims) json_coordinate_config = {json_coordinate_config}') + with open ( json_coordinate_config, 'r', encoding="utf-8") as json_cmip_coords: + cmip_coords = json.load( json_cmip_coords ) + except Exception as exc: + print(f'(check_table_dims_v_var_dims) trying to open MIP coordinate files... caught exception: exc = \n {exc}') + print(f' WARNING wont be making extra checks against accepted coordinat names...') + + dim_is_present=False + for var_dim in var_dims: + print(f'(check_table_dims_v_var_dims) var_dim = {var_dim}') + if var_dim == proj_table_var_dim: + print(f' required coordinate dimension {proj_table_var_dim} is present in the netcdf file') + dim_is_present=True + break + + if cmip_coords is not None: + print(f' did not find the exact coordinate name {proj_table_var_dim} is present in the netcdf file ') + print(f' .... checking standard/out/long names ....' ) + accepted_names = [ + cmip_coords['axis_entry'][proj_table_var_dim]['standard_name'], + cmip_coords['axis_entry'][proj_table_var_dim]['out_name'], + cmip_coords['axis_entry'][proj_table_var_dim]['long_name'] ] + print(f'(check_table_dims_v_var_dims) accepted_names = \n {accepted_names}') + if var_dim in accepted_names: + print(f' required coordinate dimension {proj_table_var_dim} is present in the netcdf file') + print(f' WARNING!!!! not an exact match for {proj_table_var_dim}.. it actually matches one of... {accepted_names}') + dim_is_present=True + break + + print(f'-----------------------------------------------------------------------------------------------------------------------------------------\n\n') + if not dim_is_present: + raise ValueError(f'(check_table_dims_v_var_dims)' + f' ERROR: dimension {proj_table_var_dim} is required for variable {target_var} / table {Path(json_table_config).name}') + print(f'(check_table_dims_v_var_dims) done comparing dimension names of between the table and the file\'s data.') + + def copy_nc(in_nc, out_nc): ''' copy target input netcdf file in_nc to target out_nc. I have to think this is not a trivial copy @@ -193,23 +248,74 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, ds = nc.Dataset(netcdf_file,'a') + # read what specific dimensions are req'd by the table for this variable + print(f'(rewrite_netcdf_file_var) attempting to read target_var dimension names FROM proj_table_vars...') + proj_table_var_dims = tuple( [ + var_dim for var_dim in proj_table_vars['variable_entry'][target_var]["dimensions"].split(' ') + ] ) + print(f' proj_table_var_dims = {proj_table_var_dims}') + print(f' type(proj_table_var_dims) = {type(proj_table_var_dims)}') + + + + + # read what specific dimensions are assoc with the variable (lat/lon/plev etc.) + print(f'(rewrite_netcdf_file_var) attempting to read target_var dimension names...') + var_dims = ds[target_var].dimensions + print(f' var_dims = {var_dims}') + + + + check_table_dims_v_var_dims( + json_table_config = json_table_config, + table_var_dims = ('latitude', 'time', 'plev19', 'longitude'),#proj_table_var_dims, + var_dims = var_dims + ) + # # check if the information req'd by the table is present in the list assoc with the variable + # print(f'(rewrite_netcdf_file_var) attempting to compare target_var dimension names with table variable entry dimension names...') + # for proj_table_var_dim in proj_table_var_dims: + # dim_is_present=False + # for var_dim in var_dims: + # if proj_table_var_dim == var_dim: + # print(f'required coordinate dimension {proj_table_var_dim} is present in the netcdf file') + # dim_is_present=True + # break + # + # if not dim_is_present: + # raise ValueError(f'(rewrite_netcdf_file_var) ERROR: dimension {proj_table_var_dim} is required for variable {target_var} / table {Path(json_table_config).name}') + # print(f'(rewrite_netcdf_file_var) done comparing dimension names of between the table and the file\'s data.') + + + # ocean grids are not implemented yet. print( '(rewrite_netcdf_file_var) checking input netcdf file for oceangrid condition') check_dataset_for_ocean_grid(ds) - # figure out the dimension names programmatically TODO - # Define lat and lon dimensions - # Assume input file is lat/lon grid - lat = ds["lat"][:] - lon = ds["lon"][:] - lat_bnds = ds["lat_bnds"][:] - lon_bnds = ds["lon_bnds"][:] - - ## Define time - #time = ds["time"][:] + # Attempt to read lat coordinates + print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lat, lat_bnds') + lat, lat_bnds = None, None + try: + lat, lat_bnds = ds["lat"][:], ds["lat_bnds"][:] + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not read latitude coordinate. moving on.\n exc = {exc}') + print(f' lat = {lat}') + print(f' lat_bnds = {lat_bnds}') + pass + + # Attempt to read lon coordinates + print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lon, lon_bnds') + lon, lon_bnds = None, None + try: + lon, lon_bnds = ds["lon"][:], ds["lon_bnds"][:] + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not read longitude coordinate. moving on.\n exc = {exc}') + print(f' lon = {lon}') + print(f' lon_bnds = {lon_bnds}') + pass # read in time_coords + units + #time = ds["time"][:] time_coords = ds["time"][:] time_coord_units = ds["time"].units print(f"(rewrite_netcdf_file_var) time_coord_units = {time_coord_units}") @@ -230,21 +336,27 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, vert_dim = get_vertical_dimension(ds, target_var) print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") - # grab var_dim - var_dim = len(var.shape) - print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}") - # Check var_dim - if var_dim not in [3, 4]: - raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.") + # grab var_N_dims and check it's values + var_N_dims = len(var.shape) + print(f"(rewrite_netcdf_file_var) var_N_dims = {var_N_dims}, local_var = {local_var}") + if var_N_dims not in [3, 4]: + raise ValueError(f"var_N_dims == {var_N_dims} != 3 nor 4. stop.") + + + print(f'(rewrite_netcdf_file_var) ASSERTING FALSE NOW') + raise Exception() - # Check var_dim and vert_dim and assign lev if relevant. - # error if vert_dim wrong given var_dim + + + + # Check var_N_dims and vert_dim and assign lev if relevant. + # error if vert_dim wrong given var_N_dims lev = None - if var_dim == 4: + if var_N_dims == 4: if vert_dim not in [ "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : - raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') + raise ValueError(f'var_N_dims={var_N_dims}, vert_dim = {vert_dim} is not supported') lev = ds[vert_dim] @@ -266,11 +378,27 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f"(rewrite_netcdf_file_var) cmor is opening json_table_config = {json_table_config}") cmor.load_table(json_table_config) + # read units units = proj_table_vars["variable_entry"] [target_var] ["units"] print(f"(rewrite_netcdf_file_var) units={units}") - cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") - cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + # setup cmor latitude axis if relevant + print(f'(rewrite_netcdf_file_var) assigning cmor_lat') + cmor_lat = None + if any( [ lat is None, lat_bnds is None ] ): + print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') + else: + cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") + + # setup cmor longitude axis if relevant + print(f'(rewrite_netcdf_file_var) assigning cmor_lon') + cmor_lon = None + if any( [ lon is None, lon_bnds is None ] ): + print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') + else: + cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + + # setup cmor_time axis if relevant try: print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" f" coord_vals = \n{time_coords}, \n" @@ -290,11 +418,27 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, ips = None # set axes for 3-dim case - if var_dim == 3: - axes = [cmor_time, cmor_lat, cmor_lon] + if var_N_dims == 3: + axes = [] + if cmor_time is not None: + axes.append(cmor_time) + else: + print(f'(rewrite_netcdf_file_var) WARNING: cmor_time is None!!! moving on...') + if cmor_lat is not None: + axes.append(cmor_lat) + else: + print(f'(rewrite_netcdf_file_var) WARNING: cmor_lat is None!!! moving on...') + if cmor_lon is not None: + axes.append(cmor_lon) + else: + print(f'(rewrite_netcdf_file_var) WARNING: cmor_lon is None!!! moving on...') print(f"(rewrite_netcdf_file_var) axes = {axes}") + + #axes2 = [cmor_time, cmor_lat, cmor_lon] + #print(f"(rewrite_netcdf_file_var) axes2 = {axes2}") + # set axes for 4-dim case - elif var_dim == 4: + elif var_N_dims == 4: if vert_dim in ["plev30", "plev19", "plev8", "height2m"]: cmor_lev = cmor.axis( vert_dim, @@ -358,6 +502,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f"(rewrite_netcdf_file_var) positive = {positive}") cmor_var = cmor.variable(target_var, units, axes, positive = positive) + # Write the output to disk #var = ds[target_var][:] #was this ever needed? why? cmor.write(cmor_var, var) diff --git a/run_test_file_cases.py b/run_test_file_cases.py index a588f803..1518c266 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -105,19 +105,20 @@ def run_cmor_RUN(filename, table, opt_var_name): # 4) FAIL (no longitude coordinate case) -# atmos, Amoon / ta +# atmos, AERmonZ / ta # just like #1, but lack longitude # Result - error, File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", line 195, in rewrite_netcdf_file_var lon = ds["lon"][:] File "src/netCDF4/_netCDF4.pyx", line 2519, in netCDF4._netCDF4.Dataset.__getitem__ IndexError: lon not found in / -testfile_atmos_gr1_AmonZ_nolons = \ +testfile_atmos_gr1_AERmonZ_nolons = \ '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/atmos_plev39_cmip.201001-201412.ta.nc' try: - some_return = run_cmor_RUN(testfile_atmos_gr1_AmonZ_nolons, 'Amon', opt_var_name = 'ta') + some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') except Exception as exc: print(f'exception caught: exc=\n{exc}') some_return=-1 pass -print_the_outcome(some_return,'atmos_gr1_AmonZ_nolons / ta') +print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') +sys.exit() ## 5) SUCCEEDS ## ocean, Omon / sos From 40129d52038841c54887039e0fb90caaf8c4a9a6 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 15 Nov 2024 11:50:03 -0500 Subject: [PATCH 02/36] wind back for a minute... --- fre/cmor/cmor_mixer.py | 195 ++++++----------------------------------- 1 file changed, 25 insertions(+), 170 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 6ba4cdf3..0f4ef243 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -20,61 +20,6 @@ # ----- \end consts ### ------ helper functions ------ ### -def check_table_dims_v_var_dims(json_table_config = None, table_var_dims = None, var_dims = None): - ''' - checks the mip table's required dimensions against the current input netcdf file's dimensions. if a req'd - dimension isn't found in the input netcdf file, either as an exact match or a match to a standard/long/out name field, - we throw an error. currently accepts three arguments, two required and one optional - json_table_config: string, optional - table_var_dims: tuple of strings, req'd - var_dims: tuple of strings, req'd - ''' - # check if the information req'd by the table is present in the list assoc with the variable - print(f'\n\n(check_table_dims_v_var_dims) attempting to compare target_var dimension names with table variable entry dimension names...') - for proj_table_var_dim in table_var_dims: - print(f'-----------------------------------------------------------------------------------------------------------------------------------------') - print(f'(check_table_dims_v_var_dims) proj_table_var_dim = {proj_table_var_dim}') - - # check the mip coordinate file for accepted standard_name, out_name, long_name flavors - cmip_coords = None - try: - json_coordinate_config = str(Path(json_table_config).parent) + '/CMIP6_coordinate.json' - print(f'(check_table_dims_v_var_dims) json_coordinate_config = {json_coordinate_config}') - with open ( json_coordinate_config, 'r', encoding="utf-8") as json_cmip_coords: - cmip_coords = json.load( json_cmip_coords ) - except Exception as exc: - print(f'(check_table_dims_v_var_dims) trying to open MIP coordinate files... caught exception: exc = \n {exc}') - print(f' WARNING wont be making extra checks against accepted coordinat names...') - - dim_is_present=False - for var_dim in var_dims: - print(f'(check_table_dims_v_var_dims) var_dim = {var_dim}') - if var_dim == proj_table_var_dim: - print(f' required coordinate dimension {proj_table_var_dim} is present in the netcdf file') - dim_is_present=True - break - - if cmip_coords is not None: - print(f' did not find the exact coordinate name {proj_table_var_dim} is present in the netcdf file ') - print(f' .... checking standard/out/long names ....' ) - accepted_names = [ - cmip_coords['axis_entry'][proj_table_var_dim]['standard_name'], - cmip_coords['axis_entry'][proj_table_var_dim]['out_name'], - cmip_coords['axis_entry'][proj_table_var_dim]['long_name'] ] - print(f'(check_table_dims_v_var_dims) accepted_names = \n {accepted_names}') - if var_dim in accepted_names: - print(f' required coordinate dimension {proj_table_var_dim} is present in the netcdf file') - print(f' WARNING!!!! not an exact match for {proj_table_var_dim}.. it actually matches one of... {accepted_names}') - dim_is_present=True - break - - print(f'-----------------------------------------------------------------------------------------------------------------------------------------\n\n') - if not dim_is_present: - raise ValueError(f'(check_table_dims_v_var_dims)' - f' ERROR: dimension {proj_table_var_dim} is required for variable {target_var} / table {Path(json_table_config).name}') - print(f'(check_table_dims_v_var_dims) done comparing dimension names of between the table and the file\'s data.') - - def copy_nc(in_nc, out_nc): ''' copy target input netcdf file in_nc to target out_nc. I have to think this is not a trivial copy @@ -248,74 +193,23 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, ds = nc.Dataset(netcdf_file,'a') - # read what specific dimensions are req'd by the table for this variable - print(f'(rewrite_netcdf_file_var) attempting to read target_var dimension names FROM proj_table_vars...') - proj_table_var_dims = tuple( [ - var_dim for var_dim in proj_table_vars['variable_entry'][target_var]["dimensions"].split(' ') - ] ) - print(f' proj_table_var_dims = {proj_table_var_dims}') - print(f' type(proj_table_var_dims) = {type(proj_table_var_dims)}') - - - - - # read what specific dimensions are assoc with the variable (lat/lon/plev etc.) - print(f'(rewrite_netcdf_file_var) attempting to read target_var dimension names...') - var_dims = ds[target_var].dimensions - print(f' var_dims = {var_dims}') - - - - check_table_dims_v_var_dims( - json_table_config = json_table_config, - table_var_dims = ('latitude', 'time', 'plev19', 'longitude'),#proj_table_var_dims, - var_dims = var_dims - ) - # # check if the information req'd by the table is present in the list assoc with the variable - # print(f'(rewrite_netcdf_file_var) attempting to compare target_var dimension names with table variable entry dimension names...') - # for proj_table_var_dim in proj_table_var_dims: - # dim_is_present=False - # for var_dim in var_dims: - # if proj_table_var_dim == var_dim: - # print(f'required coordinate dimension {proj_table_var_dim} is present in the netcdf file') - # dim_is_present=True - # break - # - # if not dim_is_present: - # raise ValueError(f'(rewrite_netcdf_file_var) ERROR: dimension {proj_table_var_dim} is required for variable {target_var} / table {Path(json_table_config).name}') - # print(f'(rewrite_netcdf_file_var) done comparing dimension names of between the table and the file\'s data.') - - - # ocean grids are not implemented yet. print( '(rewrite_netcdf_file_var) checking input netcdf file for oceangrid condition') check_dataset_for_ocean_grid(ds) - # Attempt to read lat coordinates - print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lat, lat_bnds') - lat, lat_bnds = None, None - try: - lat, lat_bnds = ds["lat"][:], ds["lat_bnds"][:] - except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING could not read latitude coordinate. moving on.\n exc = {exc}') - print(f' lat = {lat}') - print(f' lat_bnds = {lat_bnds}') - pass - - # Attempt to read lon coordinates - print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lon, lon_bnds') - lon, lon_bnds = None, None - try: - lon, lon_bnds = ds["lon"][:], ds["lon_bnds"][:] - except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING could not read longitude coordinate. moving on.\n exc = {exc}') - print(f' lon = {lon}') - print(f' lon_bnds = {lon_bnds}') - pass + # figure out the dimension names programmatically TODO + # Define lat and lon dimensions + # Assume input file is lat/lon grid + lat = ds["lat"][:] + lon = ds["lon"][:] + lat_bnds = ds["lat_bnds"][:] + lon_bnds = ds["lon_bnds"][:] - # read in time_coords + units + ## Define time #time = ds["time"][:] + + # read in time_coords + units time_coords = ds["time"][:] time_coord_units = ds["time"].units print(f"(rewrite_netcdf_file_var) time_coord_units = {time_coord_units}") @@ -336,27 +230,21 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, vert_dim = get_vertical_dimension(ds, target_var) print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") + # grab var_dim + var_dim = len(var.shape) + print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}") - # grab var_N_dims and check it's values - var_N_dims = len(var.shape) - print(f"(rewrite_netcdf_file_var) var_N_dims = {var_N_dims}, local_var = {local_var}") - if var_N_dims not in [3, 4]: - raise ValueError(f"var_N_dims == {var_N_dims} != 3 nor 4. stop.") - - - print(f'(rewrite_netcdf_file_var) ASSERTING FALSE NOW') - raise Exception() + # Check var_dim + if var_dim not in [3, 4]: + raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.") - - - - # Check var_N_dims and vert_dim and assign lev if relevant. - # error if vert_dim wrong given var_N_dims + # Check var_dim and vert_dim and assign lev if relevant. + # error if vert_dim wrong given var_dim lev = None - if var_N_dims == 4: + if var_dim == 4: if vert_dim not in [ "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : - raise ValueError(f'var_N_dims={var_N_dims}, vert_dim = {vert_dim} is not supported') + raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') lev = ds[vert_dim] @@ -378,27 +266,11 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f"(rewrite_netcdf_file_var) cmor is opening json_table_config = {json_table_config}") cmor.load_table(json_table_config) - # read units units = proj_table_vars["variable_entry"] [target_var] ["units"] print(f"(rewrite_netcdf_file_var) units={units}") - # setup cmor latitude axis if relevant - print(f'(rewrite_netcdf_file_var) assigning cmor_lat') - cmor_lat = None - if any( [ lat is None, lat_bnds is None ] ): - print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') - else: - cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") - - # setup cmor longitude axis if relevant - print(f'(rewrite_netcdf_file_var) assigning cmor_lon') - cmor_lon = None - if any( [ lon is None, lon_bnds is None ] ): - print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') - else: - cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") - - # setup cmor_time axis if relevant + cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") + cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") try: print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" f" coord_vals = \n{time_coords}, \n" @@ -418,27 +290,11 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, ips = None # set axes for 3-dim case - if var_N_dims == 3: - axes = [] - if cmor_time is not None: - axes.append(cmor_time) - else: - print(f'(rewrite_netcdf_file_var) WARNING: cmor_time is None!!! moving on...') - if cmor_lat is not None: - axes.append(cmor_lat) - else: - print(f'(rewrite_netcdf_file_var) WARNING: cmor_lat is None!!! moving on...') - if cmor_lon is not None: - axes.append(cmor_lon) - else: - print(f'(rewrite_netcdf_file_var) WARNING: cmor_lon is None!!! moving on...') + if var_dim == 3: + axes = [cmor_time, cmor_lat, cmor_lon] print(f"(rewrite_netcdf_file_var) axes = {axes}") - - #axes2 = [cmor_time, cmor_lat, cmor_lon] - #print(f"(rewrite_netcdf_file_var) axes2 = {axes2}") - # set axes for 4-dim case - elif var_N_dims == 4: + elif var_dim == 4: if vert_dim in ["plev30", "plev19", "plev8", "height2m"]: cmor_lev = cmor.axis( vert_dim, @@ -502,7 +358,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f"(rewrite_netcdf_file_var) positive = {positive}") cmor_var = cmor.variable(target_var, units, axes, positive = positive) - # Write the output to disk #var = ds[target_var][:] #was this ever needed? why? cmor.write(cmor_var, var) From 5fb52ceb804edf8d2355024b7f06282e403808e3 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 15 Nov 2024 13:39:01 -0500 Subject: [PATCH 03/36] zonally averaged case AERmonZ / ta, aka CASE 4 now succeeds. objects assigned None and checked if corresponding cmor axis should bother to be assigned, instead of relying on dimenionality of the data --- fre/cmor/cmor_mixer.py | 140 +++++++++++++++++++++++++++---------- run_test_file_cases.py | 152 +++++++++++++++++++++++------------------ 2 files changed, 192 insertions(+), 100 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 0f4ef243..2de8d570 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -198,24 +198,48 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, check_dataset_for_ocean_grid(ds) - # figure out the dimension names programmatically TODO - # Define lat and lon dimensions - # Assume input file is lat/lon grid - lat = ds["lat"][:] - lon = ds["lon"][:] - lat_bnds = ds["lat_bnds"][:] - lon_bnds = ds["lon_bnds"][:] + # try to read what coordinate(s) we're going to be expecting for the variable + expected_mip_coord_dims=None + try: + expected_mip_coord_dims = proj_table_vars["variable_entry"] [target_var] ["dimensions"] + print( '(rewrite_netcdf_file_var) i am hoping to find data for the following coordinate dimensions:\n' + f' expected_mip_coord_dims = {expected_mip_coord_dims}' ) + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not get expected coordinate dimensions for {target_var}. ' + ' in proj_table_vars file {json_table_config}. \n exc = {exc}') + - ## Define time - #time = ds["time"][:] + ## figure out the coordinate/dimension names programmatically TODO + + # Attempt to read lat coordinates + print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lat, lat_bnds') + lat, lat_bnds = None, None + try: + lat, lat_bnds = ds["lat"][:], ds["lat_bnds"][:] + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not read latitude coordinate. moving on.\n exc = {exc}') + print(f' lat = {lat}') + print(f' lat_bnds = {lat_bnds}') + pass + + # Attempt to read lon coordinates + print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lon, lon_bnds') + lon, lon_bnds = None, None + try: + lon, lon_bnds = ds["lon"][:], ds["lon_bnds"][:] + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not read longitude coordinate. moving on.\n exc = {exc}') + print(f' lon = {lon}') + print(f' lon_bnds = {lon_bnds}') + pass # read in time_coords + units - time_coords = ds["time"][:] + time_coords = ds["time"][:] # out this in a try/except thingy, initializing like others? time_coord_units = ds["time"].units print(f"(rewrite_netcdf_file_var) time_coord_units = {time_coord_units}") # read in time_bnds , if present - time_bnds = [] + time_bnds = [] # shouldnt this be initialized like the others? try: time_bnds = ds["time_bnds"][:] #print(f"(rewrite_netcdf_file_var) time_bnds = {time_bnds}") @@ -226,10 +250,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # read the input variable data, i believe var = ds[target_var][:] - # determine the vertical dimension by looping over netcdf variables - vert_dim = get_vertical_dimension(ds, target_var) - print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") - # grab var_dim var_dim = len(var.shape) print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}") @@ -238,11 +258,17 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, if var_dim not in [3, 4]: raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.") + # determine the vertical dimension by looping over netcdf variables + vert_dim = get_vertical_dimension(ds, target_var) # returns int( 0 ) if not present + print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") + # Check var_dim and vert_dim and assign lev if relevant. # error if vert_dim wrong given var_dim lev = None - if var_dim == 4: - if vert_dim not in [ "plev30", "plev19", "plev8", + #if var_dim == 4: + # if vert_dim not in [ "plev30", "plev19", "plev8", + if vert_dim != 0: + if vert_dim not in [ "plev39", "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') lev = ds[vert_dim] @@ -269,8 +295,27 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, units = proj_table_vars["variable_entry"] [target_var] ["units"] print(f"(rewrite_netcdf_file_var) units={units}") - cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") - cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + + # setup cmor latitude axis if relevant + #cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") + #cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + print(f'(rewrite_netcdf_file_var) assigning cmor_lat') + cmor_lat = None + if any( [ lat is None, lat_bnds is None ] ): + print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') + else: + cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") + + # setup cmor longitude axis if relevant + print(f'(rewrite_netcdf_file_var) assigning cmor_lon') + cmor_lon = None + if any( [ lon is None, lon_bnds is None ] ): + print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') + else: + cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + + # setup cmor time axis if relevant + cmor_time = None try: print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" f" coord_vals = \n{time_coords}, \n" @@ -283,20 +328,18 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, " coord_vals = time_coords, units = time_coord_units)") cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) - # initializations + + + # other vertical-axis-relevant initializations save_ps = False ps = None ierr_ap, ierr_b = None, None ips = None - # set axes for 3-dim case - if var_dim == 3: - axes = [cmor_time, cmor_lat, cmor_lon] - print(f"(rewrite_netcdf_file_var) axes = {axes}") - # set axes for 4-dim case - elif var_dim == 4: - - if vert_dim in ["plev30", "plev19", "plev8", "height2m"]: + # set cmor vertical axis if relevant + cmor_lev = None + if lev is not None: + if vert_dim in ["plev39", "plev30", "plev19", "plev8", "height2m"]: cmor_lev = cmor.axis( vert_dim, coord_vals = lev[:], units = lev.units ) @@ -342,15 +385,44 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) ierr_ap after calling cmor_zfactor: {ierr_ap}\n' f'(rewrite_netcdf_file_var) ierr_b after calling cmor_zfactor: {ierr_b}' ) + axis_ids = [] + if cmor_time is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_time to axis_ids list...') + axis_ids.append(cmor_time) + print(f' axis_ids now = {axis_ids}') + if cmor_lat is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lat to axis_ids list...') + axis_ids.append(cmor_lat) + print(f' axis_ids now = {axis_ids}') + if cmor_lon is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lon to axis_ids list...') + axis_ids.append(cmor_lon) + print(f' axis_ids now = {axis_ids}') + ips = cmor.zfactor( zaxis_id = cmor_lev, zfactor_name = "ps", - axis_ids = [cmor_time, cmor_lat, cmor_lon], + axis_ids = axis_ids, #[cmor_time, cmor_lat, cmor_lon], units = "Pa" ) save_ps = True - # assign axes at end of 4-dim case - axes = [cmor_time, cmor_lev, cmor_lat, cmor_lon] - - + + + axes = [] + if cmor_time is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_time to axes list...') + axes.append(cmor_time) + print(f' axes now = {axes}') + if cmor_lev is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lev to axes list...') + axes.append(cmor_lev) + print(f' axes now = {axes}') + if cmor_lat is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lat to axes list...') + axes.append(cmor_lat) + print(f' axes now = {axes}') + if cmor_lon is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_lon to axes list...') + axes.append(cmor_lon) + print(f' axes now = {axes}') # read positive attribute and create cmor_var? can this return none? TODO @@ -359,7 +431,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor_var = cmor.variable(target_var, units, axes, positive = positive) # Write the output to disk - #var = ds[target_var][:] #was this ever needed? why? cmor.write(cmor_var, var) if save_ps: if any( [ ips is None, ps is None ] ): @@ -371,7 +442,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor.close(ips, file_name = True, preserve = False) filename = cmor.close(cmor_var, file_name = True, preserve = False) print(f"(rewrite_netcdf_file_var) returned by cmor.close: filename = {filename}") - #cmor.close() ds.close() print('-------------------------- END rewrite_netcdf_file_var call -----\n\n') diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 1518c266..7f3fdb34 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -65,51 +65,80 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return -## 1) SUCCEEDs -## land, Lmon, gr1 -#testfile_land_gr1_Lmon = \ -# '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/pp/land/ts/monthly/5yr/land.005101-005512.lai.nc' -#try: -# some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -#except: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'land_gr1_Lmon / lai') +# 8) FAIL (no latitude nor longitude coordinates cases) +# atmos, Amon / ch4global +# Result - error, +# File "src/netCDF4/_netCDF4.pyx", line 2519, in netCDF4._netCDF4.Dataset.__getitem__ IndexError: +# lat not found in / +testfile_atmos_scalar_gn_Amon_nolon_nolat = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_scalar/ts/monthly/5yr/' + \ + 'atmos_scalar.197001-197412.ch4global.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') -## 2) SUCCEEDs -## atmos, Amon / cl -#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_level_cmip/ts/monthly/5yr/atmos_level_cmip.196001-196412.cl.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') +sys.exit() -## 3) SUCCEEDs -## atmos, Amon / mc -#testfile_atmos_level_cmip_gr1_Amon_fullL = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_level_cmip/ts/monthly/5yr/atmos_level_cmip.195501-195912.mc.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') +# 1) SUCCEEDs +# land, Lmon, gr1 +testfile_land_gr1_Lmon = \ + '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/land/ts/monthly/5yr/' + \ + 'land.005101-005512.lai.nc' +try: + some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') +except: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'land_gr1_Lmon / lai') + + +# 2) SUCCEEDs +# atmos, Amon / cl +testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.196001-196412.cl.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') + + +# 3) SUCCEEDs +# atmos, Amon / mc +testfile_atmos_level_cmip_gr1_Amon_fullL = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.195501-195912.mc.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') + -# 4) FAIL (no longitude coordinate case) +# 4) SUCCEEDs (no longitude coordinate case) # atmos, AERmonZ / ta # just like #1, but lack longitude -# Result - error, File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", line 195, in rewrite_netcdf_file_var lon = ds["lon"][:] File "src/netCDF4/_netCDF4.pyx", line 2519, in netCDF4._netCDF4.Dataset.__getitem__ IndexError: lon not found in / testfile_atmos_gr1_AERmonZ_nolons = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/atmos_plev39_cmip.201001-201412.ta.nc' + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ + 'atmos_plev39_cmip.201001-201412.ta.nc' try: some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') except Exception as exc: @@ -118,19 +147,20 @@ def run_cmor_RUN(filename, table, opt_var_name): pass print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') -sys.exit() -## 5) SUCCEEDS -## ocean, Omon / sos -#testfile_ocean_monthly_1x1deg_gr = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/ocean_monthly_1x1deg/ts/monthly/5yr/ocean_monthly_1x1deg.190001-190412.sos.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') +# 5) SUCCEEDs +# ocean, Omon / sos +testfile_ocean_monthly_1x1deg_gr = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_1x1deg.190001-190412.sos.nc' +try: + some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') @@ -138,7 +168,9 @@ def run_cmor_RUN(filename, table, opt_var_name): ## ocean, Omon / sos ## Result - error, AttributeError: NetCDF: Attempt to define fill value when data already exists. #testfile_ocean_monthly_gn = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/pp/ocean_monthly/ts/monthly/5yr/ocean_monthly.002101-002512.sos.nc' +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly/ts/monthly/5yr/' + \ +# 'ocean_monthly.002101-002512.sos.nc' #try: # some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') #except Exception as exc: @@ -153,7 +185,9 @@ def run_cmor_RUN(filename, table, opt_var_name): ## ocean, Omon / so ## Result - identical failure to #6 #testfile_ocean_monthly_z_1x1deg_gr = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/ocean_monthly_z_1x1deg.000101-000512.so.nc' +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ +# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' #try: # some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') #except Exception as exc: @@ -163,27 +197,15 @@ def run_cmor_RUN(filename, table, opt_var_name): #print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') -# 8) FAIL (no latitude nor longitude coordinates cases) -# atmos, Amon / ch4global -# Result - error, File "src/netCDF4/_netCDF4.pyx", line 2519, in netCDF4._netCDF4.Dataset.__getitem__ IndexError: lat not found in / -testfile_atmos_scalar_gn_Amon_nolon_nolat = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/atmos_scalar/ts/monthly/5yr/atmos_scalar.197001-197412.ch4global.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - - ## 9) FAIL (4 dimensional data with no vertical) ## Result - error, ## File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", ## line 134, in get_vertical_dimension if not (ds[dim].axis and ds[dim].axis == "Z"): ## AttributeError: NetCDF: Attribute not found #testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp/LUmip_refined/ts/monthly/5yr/LUmip_refined.185001-185412.gppLut.nc' +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' #try: # some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') #except Exception as exc: From 30adf0fcc47f63d77f4877ef707e6f2ee5d43648 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 15 Nov 2024 13:52:06 -0500 Subject: [PATCH 04/36] atmos_scalar data now works- case8! this case works with only time coordinate-depdendent data! --- fre/cmor/cmor_mixer.py | 17 +++++------ run_test_file_cases.py | 66 ++++++++++++++++++++---------------------- 2 files changed, 38 insertions(+), 45 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 2de8d570..a6a315c6 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -254,9 +254,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, var_dim = len(var.shape) print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}") - # Check var_dim - if var_dim not in [3, 4]: - raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.") + ## Check var_dim + #if var_dim not in [3, 4]: + # raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.") # determine the vertical dimension by looping over netcdf variables vert_dim = get_vertical_dimension(ds, target_var) # returns int( 0 ) if not present @@ -265,8 +265,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # Check var_dim and vert_dim and assign lev if relevant. # error if vert_dim wrong given var_dim lev = None - #if var_dim == 4: - # if vert_dim not in [ "plev30", "plev19", "plev8", if vert_dim != 0: if vert_dim not in [ "plev39", "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : @@ -297,21 +295,19 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # setup cmor latitude axis if relevant - #cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") - #cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") - print(f'(rewrite_netcdf_file_var) assigning cmor_lat') cmor_lat = None if any( [ lat is None, lat_bnds is None ] ): print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') else: + print(f'(rewrite_netcdf_file_var) assigning cmor_lat') cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") # setup cmor longitude axis if relevant - print(f'(rewrite_netcdf_file_var) assigning cmor_lon') cmor_lon = None if any( [ lon is None, lon_bnds is None ] ): print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') else: + print(f'(rewrite_netcdf_file_var) assigning cmor_lon') cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") # setup cmor time axis if relevant @@ -320,15 +316,16 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" f" coord_vals = \n{time_coords}, \n" f" cell_bounds = time_bnds, units = {time_coord_units}) ") + print(f'(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...') cmor_time = cmor.axis("time", coord_vals = time_coords, cell_bounds = time_bnds, units = time_coord_units) except ValueError as exc: print(f"(rewrite_netcdf_file_var) WARNING exception raised... exc={exc}\n" " cmor_time = cmor.axis('time', \n" " coord_vals = time_coords, units = time_coord_units)") + print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) - # other vertical-axis-relevant initializations save_ps = False diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 7f3fdb34..e4659b35 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -65,25 +65,23 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return -# 8) FAIL (no latitude nor longitude coordinates cases) -# atmos, Amon / ch4global -# Result - error, -# File "src/netCDF4/_netCDF4.pyx", line 2519, in netCDF4._netCDF4.Dataset.__getitem__ IndexError: -# lat not found in / -testfile_atmos_scalar_gn_Amon_nolon_nolat = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_scalar/ts/monthly/5yr/' + \ - 'atmos_scalar.197001-197412.ch4global.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - - -sys.exit() +## 9) FAIL (4 dimensional data with no vertical) +## Result - error, +## File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", +## line 134, in get_vertical_dimension if not (ds[dim].axis and ds[dim].axis == "Z"): +## AttributeError: NetCDF: Attribute not found +#testfile_LUmip_refined_gr1_Emon_landusedim = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' +#try: +# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') +#sys.exit() # 1) SUCCEEDs @@ -197,22 +195,20 @@ def run_cmor_RUN(filename, table, opt_var_name): #print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') -## 9) FAIL (4 dimensional data with no vertical) -## Result - error, -## File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", -## line 134, in get_vertical_dimension if not (ds[dim].axis and ds[dim].axis == "Z"): -## AttributeError: NetCDF: Attribute not found -#testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/LUmip_refined/ts/monthly/5yr/' + \ -# 'LUmip_refined.185001-185412.gppLut.nc' -#try: -# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') +# 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) +# atmos, Amon / ch4global +testfile_atmos_scalar_gn_Amon_nolon_nolat = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_scalar/ts/monthly/5yr/' + \ + 'atmos_scalar.197001-197412.ch4global.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') + From fe595110faf1b7a794e0d6fddb5e39b4c78919bd Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 15 Nov 2024 16:05:20 -0500 Subject: [PATCH 05/36] landuse giving issues with gppLut- --- fre/cmor/cmor_mixer.py | 72 ++++++++++++++++++++++++++++++++---------- run_test_file_cases.py | 34 ++++++++++---------- 2 files changed, 72 insertions(+), 34 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index a6a315c6..c544faf7 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -130,8 +130,16 @@ def get_vertical_dimension(ds,target_var): # if it is not a vertical axis, move on. print(f'(get_vertical_dimension) dim={dim}') if dim == 'landuse': - continue + #continue + print(f'(get_vertical_dimension) i think i will crash... NOW') + vert_dim = dim + break + + if dim == 'landuse': + print(f'(get_vertical_dimension) i think i will crash... NOW') if not (ds[dim].axis and ds[dim].axis == "Z"): + if dim == 'landuse': + print(f'(get_vertical_dimension) I WILL NOT SEE THIS PRINTOUT!!!') continue vert_dim = dim return vert_dim @@ -221,6 +229,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f' lat = {lat}') print(f' lat_bnds = {lat_bnds}') pass + print(f' DONE attempting to read coordinate(s), lat, lat_bnds') # Attempt to read lon coordinates print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lon, lon_bnds') @@ -232,11 +241,13 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f' lon = {lon}') print(f' lon_bnds = {lon_bnds}') pass - + print(f' DONE attempting to read coordinate(s), lon, lon_bnds') + # read in time_coords + units + print(f'(rewrite_netcdf_file_var) attempting to read time_coords, and units...') time_coords = ds["time"][:] # out this in a try/except thingy, initializing like others? time_coord_units = ds["time"].units - print(f"(rewrite_netcdf_file_var) time_coord_units = {time_coord_units}") + print(f" time_coord_units = {time_coord_units}") # read in time_bnds , if present time_bnds = [] # shouldnt this be initialized like the others? @@ -264,21 +275,22 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # Check var_dim and vert_dim and assign lev if relevant. # error if vert_dim wrong given var_dim - lev = None + lev, lev_units = None, "" if vert_dim != 0: - if vert_dim not in [ "plev39", "plev30", "plev19", "plev8", + if vert_dim not in [ "landuse", "plev39", "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') lev = ds[vert_dim] - + if vert_dim != "landuse": + lev_units = ds[vert_dim].units # now we set up the cmor module object # initialize CMOR cmor.setup( netcdf_file_action = cmor.CMOR_PRESERVE, - set_verbosity = cmor.CMOR_QUIET, #default is CMOR_NORMAL + set_verbosity = cmor.CMOR_NORMAL, #CMOR_QUIET, #default is CMOR_NORMAL exit_control = cmor.CMOR_NORMAL, - logfile = None, + logfile = './foo.log', create_subdirectories = 1 ) @@ -301,6 +313,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, else: print(f'(rewrite_netcdf_file_var) assigning cmor_lat') cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") + print(f' DONE assigning cmor_lat') # setup cmor longitude axis if relevant cmor_lon = None @@ -309,22 +322,41 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, else: print(f'(rewrite_netcdf_file_var) assigning cmor_lon') cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + print(f' DONE assigning cmor_lon') # setup cmor time axis if relevant cmor_time = None - try: + print(f'(rewrite_netcdf_file_var) assigning cmor_time') + try: #if vert_dim != 'landuse': print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" f" coord_vals = \n{time_coords}, \n" f" cell_bounds = time_bnds, units = {time_coord_units}) ") - print(f'(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...') + print(f'(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...') cmor_time = cmor.axis("time", coord_vals = time_coords, cell_bounds = time_bnds, units = time_coord_units) - except ValueError as exc: - print(f"(rewrite_netcdf_file_var) WARNING exception raised... exc={exc}\n" - " cmor_time = cmor.axis('time', \n" + except ValueError as exc: #else: + print(f"(rewrite_netcdf_file_var) cmor_time = cmor.axis('time', \n" " coord_vals = time_coords, units = time_coord_units)") print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) + print(f' DONE assigning cmor_time') + +# # setup cmor time axis if relevant +# cmor_time = None +# try: +# print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" +# f" coord_vals = \n{time_coords}, \n" +# f" cell_bounds = time_bnds, units = {time_coord_units}) ") +# print(f'(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...') +# cmor_time = cmor.axis("time", coord_vals = time_coords, +# cell_bounds = time_bnds, units = time_coord_units) +# except ValueError as exc: +# print(f"(rewrite_netcdf_file_var) WARNING exception raised... exc={exc}\n" +# " cmor_time = cmor.axis('time', \n" +# " coord_vals = time_coords, units = time_coord_units)") +# print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') +# cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) + # other vertical-axis-relevant initializations @@ -336,9 +368,14 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # set cmor vertical axis if relevant cmor_lev = None if lev is not None: - if vert_dim in ["plev39", "plev30", "plev19", "plev8", "height2m"]: + print(f'(rewrite_netcdf_file_var) assigning cmor_lev') + if vert_dim in ["landuse", "plev39", "plev30", "plev19", "plev8", "height2m"]: + print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') + cmor_vert_dim_name = vert_dim + if vert_dim == "landuse": + cmor_vert_dim_name = "landUse" # this is why can't we have nice things cmor_lev = cmor.axis( vert_dim, - coord_vals = lev[:], units = lev.units ) + coord_vals = lev[:], units = lev_units ) elif vert_dim in ["level", "lev", "levhalf"]: # find the ps file nearby @@ -351,7 +388,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, if vert_dim == "levhalf": cmor_lev = cmor.axis( "alternate_hybrid_sigma_half", coord_vals = lev[:], - units = lev.units ) + units = lev_units ) ierr_ap = cmor.zfactor( zaxis_id = cmor_lev, zfactor_name = "ap_half", axis_ids = [cmor_lev, ], @@ -365,7 +402,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, else: cmor_lev = cmor.axis( "alternate_hybrid_sigma", coord_vals = lev[:], - units = lev.units, + units = lev_units, cell_bounds = ds[vert_dim+"_bnds"] ) ierr_ap = cmor.zfactor( zaxis_id = cmor_lev, zfactor_name = "ap", @@ -401,6 +438,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, axis_ids = axis_ids, #[cmor_time, cmor_lat, cmor_lon], units = "Pa" ) save_ps = True + print(f' DONE assigning cmor_lev') axes = [] diff --git a/run_test_file_cases.py b/run_test_file_cases.py index e4659b35..d4496d9f 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -65,23 +65,23 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return -## 9) FAIL (4 dimensional data with no vertical) -## Result - error, -## File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", -## line 134, in get_vertical_dimension if not (ds[dim].axis and ds[dim].axis == "Z"): -## AttributeError: NetCDF: Attribute not found -#testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/LUmip_refined/ts/monthly/5yr/' + \ -# 'LUmip_refined.185001-185412.gppLut.nc' -#try: -# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') -#sys.exit() +# 9) FAIL (4 dimensional data with no vertical) +# Result - error, +# File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", +# line 134, in get_vertical_dimension if not (ds[dim].axis and ds[dim].axis == "Z"): +# AttributeError: NetCDF: Attribute not found +testfile_LUmip_refined_gr1_Emon_landusedim = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/LUmip_refined/ts/monthly/5yr/' + \ + 'LUmip_refined.185001-185412.gppLut.nc' +try: + some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') +sys.exit() # 1) SUCCEEDs From 8672155c1aa7e6f53a8b9bec1fc3ddc758d1d96f Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 15 Nov 2024 17:13:27 -0500 Subject: [PATCH 06/36] wow this LUmip file.... case9 is gonna be a fun one --- fre/cmor/cmor_mixer.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index c544faf7..64564b10 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -144,7 +144,7 @@ def get_vertical_dimension(ds,target_var): vert_dim = dim return vert_dim -def create_tmp_dir(outdir): +def create_tmp_dir(outdir, json_exp_config = None): ''' creates a tmp_dir based on targeted output directory root. returns the name of the tmp dir. accepts one argument: @@ -152,7 +152,20 @@ def create_tmp_dir(outdir): file output. tmp_dir will be slightly different depending on the output directory targeted ''' + outdir_from_exp_config=None + if json_exp_config is not None: + with open(json_exp_config, "r", encoding = "utf-8") as table_config_file: + try: + print(' FOO') + exp_config = json.load(table_config_file) + print(' BAR') + outdir_from_exp_config = exp_config["outpath"] + except: + print(f'(create_tmp_dir) could not read outdir from json_exp_config... oh well!') + + print(f"(create_tmp_dir) outdir = {outdir}") + assert False tmp_dir = None if any( [ outdir == "/local2", outdir.find("/work") != -1, @@ -164,6 +177,8 @@ def create_tmp_dir(outdir): tmp_dir = str( Path(f"{outdir}/tmp/").resolve() ) try: os.makedirs(tmp_dir, exist_ok=True) + if outdir_from_exp_config is not None: + os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok=True) except Exception as exc: raise OSError('(create_tmp_dir) problem creating temp output directory. stop.') from exc return tmp_dir @@ -275,7 +290,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # Check var_dim and vert_dim and assign lev if relevant. # error if vert_dim wrong given var_dim - lev, lev_units = None, "" + lev, lev_units = None, "1" #1 #"none" #None #"" if vert_dim != 0: if vert_dim not in [ "landuse", "plev39", "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : @@ -289,7 +304,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor.setup( netcdf_file_action = cmor.CMOR_PRESERVE, set_verbosity = cmor.CMOR_NORMAL, #CMOR_QUIET, #default is CMOR_NORMAL - exit_control = cmor.CMOR_NORMAL, + exit_control = cmor.CMOR_EXIT_ON_WARNING,#CMOR_NORMAL, logfile = './foo.log', create_subdirectories = 1 ) @@ -374,7 +389,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor_vert_dim_name = vert_dim if vert_dim == "landuse": cmor_vert_dim_name = "landUse" # this is why can't we have nice things - cmor_lev = cmor.axis( vert_dim, + cmor_lev = cmor.axis( cmor_vert_dim_name, coord_vals = lev[:], units = lev_units ) elif vert_dim in ["level", "lev", "levhalf"]: @@ -514,7 +529,7 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None, #determine a tmp dir for working on files. - tmp_dir = create_tmp_dir( outdir ) + '/' + tmp_dir = create_tmp_dir( outdir, json_exp_config) + '/' print(f'(cmorize_target_var_files) will use tmp_dir={tmp_dir}') From 240c32e6c2531cf0df281864d507f8168a7dae48 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 18 Nov 2024 15:56:44 -0500 Subject: [PATCH 07/36] fix json syntax mistake that thankfully does not seem to bother the cmor module at all --- fre/tests/test_files/CMOR_input_example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/tests/test_files/CMOR_input_example.json b/fre/tests/test_files/CMOR_input_example.json index 3ce8a985..f37b85e1 100644 --- a/fre/tests/test_files/CMOR_input_example.json +++ b/fre/tests/test_files/CMOR_input_example.json @@ -70,5 +70,5 @@ "#output_path_template": "Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax", "output_path_template": "<_member_id>", - "output_file_template": "
<_member_id>", + "output_file_template": "
<_member_id>" } From 4f53f895934c87f186fbdf7190052da32318563e Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 18 Nov 2024 15:57:32 -0500 Subject: [PATCH 08/36] point run_test_file_cases at ocean files... the land file will wait a little longer --- run_test_file_cases.py | 239 +++++++++++++++++++++-------------------- 1 file changed, 125 insertions(+), 114 deletions(-) diff --git a/run_test_file_cases.py b/run_test_file_cases.py index d4496d9f..d031c2e4 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -65,151 +65,162 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return -# 9) FAIL (4 dimensional data with no vertical) -# Result - error, -# File "/home/Ian.Laflotte/Working/fre-cli/fre/cmor/cmor_mixer.py", -# line 134, in get_vertical_dimension if not (ds[dim].axis and ds[dim].axis == "Z"): -# AttributeError: NetCDF: Attribute not found -testfile_LUmip_refined_gr1_Emon_landusedim = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/LUmip_refined/ts/monthly/5yr/' + \ - 'LUmip_refined.185001-185412.gppLut.nc' -try: - some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') -sys.exit() - - -# 1) SUCCEEDs -# land, Lmon, gr1 -testfile_land_gr1_Lmon = \ - '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/land/ts/monthly/5yr/' + \ - 'land.005101-005512.lai.nc' -try: - some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -except: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'land_gr1_Lmon / lai') - - -# 2) SUCCEEDs -# atmos, Amon / cl -testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.196001-196412.cl.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') - - -# 3) SUCCEEDs -# atmos, Amon / mc -testfile_atmos_level_cmip_gr1_Amon_fullL = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.195501-195912.mc.nc' +# 6) FAIL (copy_nc failure!!! WEIRD) +# ocean, Omon / sos +# Result - error, AttributeError: NetCDF: Attempt to define fill value when data already exists. +testfile_ocean_monthly_gn = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly/ts/monthly/5yr/' + \ + 'ocean_monthly.002101-002512.sos.nc' try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') + some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') except Exception as exc: print(f'exception caught: exc=\n{exc}') some_return=-1 pass -print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') +print_the_outcome(some_return,'ocean_monthly_gn / sos') +if some_return != 0: + print('didnt pass ocean-file test number 1... exit.') + sys.exit() -# 4) SUCCEEDs (no longitude coordinate case) -# atmos, AERmonZ / ta -# just like #1, but lack longitude -testfile_atmos_gr1_AERmonZ_nolons = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ - 'atmos_plev39_cmip.201001-201412.ta.nc' +# 7) FAIL (copy_nc failure!!! WEIRD) +# ocean, Omon / so +# Result - identical failure to #6 +testfile_ocean_monthly_z_1x1deg_gr = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_z_1x1deg.000101-000512.so.nc' try: - some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') + some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') except Exception as exc: print(f'exception caught: exc=\n{exc}') some_return=-1 pass -print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') - - -# 5) SUCCEEDs -# ocean, Omon / sos -testfile_ocean_monthly_1x1deg_gr = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_1x1deg.190001-190412.sos.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') - - - -## 6) FAIL (copy_nc failure!!! WEIRD) +print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') +if some_return != 0: + print('didnt pass ocean-file test number 2... exit.') + sys.exit() + + +## 1) SUCCEEDs +## land, Lmon, gr1 +#testfile_land_gr1_Lmon = \ +# '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/land/ts/monthly/5yr/' + \ +# 'land.005101-005512.lai.nc' +#try: +# some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') +#except: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'land_gr1_Lmon / lai') +# +# +## 2) SUCCEEDs +## atmos, Amon / cl +#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ +# 'atmos_level_cmip.196001-196412.cl.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') +# +# +## 3) SUCCEEDs +## atmos, Amon / mc +#testfile_atmos_level_cmip_gr1_Amon_fullL = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ +# 'atmos_level_cmip.195501-195912.mc.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') +# +# +# +## 4) SUCCEEDs (no longitude coordinate case) +## atmos, AERmonZ / ta +## just like #1, but lack longitude +#testfile_atmos_gr1_AERmonZ_nolons = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ +# 'atmos_plev39_cmip.201001-201412.ta.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') +# +# +## 5) SUCCEEDs ## ocean, Omon / sos -## Result - error, AttributeError: NetCDF: Attempt to define fill value when data already exists. -#testfile_ocean_monthly_gn = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/ocean_monthly/ts/monthly/5yr/' + \ -# 'ocean_monthly.002101-002512.sos.nc' +#testfile_ocean_monthly_1x1deg_gr = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ +# 'ocean_monthly_1x1deg.190001-190412.sos.nc' #try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') #except Exception as exc: # print(f'exception caught: exc=\n{exc}') # some_return=-1 # pass -#print_the_outcome(some_return,'ocean_monthly_gn / sos') - - - -## 7) FAIL (copy_nc failure!!! WEIRD) -## ocean, Omon / so -## Result - identical failure to #6 -#testfile_ocean_monthly_z_1x1deg_gr = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ -# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' +#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') +# +# +# +# +## 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) +## atmos, Amon / ch4global +#testfile_atmos_scalar_gn_Amon_nolon_nolat = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_scalar/ts/monthly/5yr/' + \ +# 'atmos_scalar.197001-197412.ch4global.nc' #try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') +# some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') #except Exception as exc: # print(f'exception caught: exc=\n{exc}') # some_return=-1 # pass -#print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') +#print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') + + -# 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) -# atmos, Amon / ch4global -testfile_atmos_scalar_gn_Amon_nolon_nolat = \ + +# 9) FAIL (4 dimensional data with no vertical) +# Result - error, +# the variable gppLut needs coordinate variables time lat lon and landuse +# the landuse coordinate is expected to be of character type, and one of four different specific strings, +# each string representing primary/secondary, pasture, crops, urban style land usage. +# this file's landuse coordinate is an integer between 0 and 3, so it's not clear to the CMOR module +# how to map the integers to the string values (though it's obvious to me) +testfile_LUmip_refined_gr1_Emon_landusedim = \ '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_scalar/ts/monthly/5yr/' + \ - 'atmos_scalar.197001-197412.ch4global.nc' + 'pp/LUmip_refined/ts/monthly/5yr/' + \ + 'LUmip_refined.185001-185412.gppLut.nc' try: - some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') + some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') except Exception as exc: print(f'exception caught: exc=\n{exc}') some_return=-1 pass -print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - - - +print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') +if some_return != 0: + print('didnt pass the land-file test. exit.') + sys.exit() From d43dfc31b923fa7c852042bdd688a50696ce7f8f Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 18 Nov 2024 15:59:06 -0500 Subject: [PATCH 09/36] now the exp_config_json file gets checked for the output field, and the code tries to create this. this is to avoid a warning from the CMOR module. it might be good to be able to do all of this so well that we could error on warning some day. ... oither than that, not much other than keeping certain other things around like CMOR behavior field values --- fre/cmor/cmor_mixer.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 64564b10..5f5f515f 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -112,13 +112,12 @@ def check_dataset_for_ocean_grid(ds): "(check_dataset_for_ocean_grid) 'xh' found in var_list. ocean grid req'd but not yet unimplemented. stop.") -def get_vertical_dimension(ds,target_var): +def get_vertical_dimension(ds, target_var): ''' determines the vertical dimensionality of target_var within netCDF4 Dataset ds. accepts two arguments and returns an object represnting the vertical dimensions assoc with the target_var. ds: netCDF4.Dataset object containing variables with associated dimensional information. target_var: string, representating a variable contained within the netCDF4.Dataset ds - ''' vert_dim = 0 for name, variable in ds.variables.items(): @@ -152,20 +151,16 @@ def create_tmp_dir(outdir, json_exp_config = None): file output. tmp_dir will be slightly different depending on the output directory targeted ''' - outdir_from_exp_config=None + outdir_from_exp_config = None if json_exp_config is not None: with open(json_exp_config, "r", encoding = "utf-8") as table_config_file: try: - print(' FOO') - exp_config = json.load(table_config_file) - print(' BAR') - outdir_from_exp_config = exp_config["outpath"] + outdir_from_exp_config = json.load(table_config_file)["outpath"] except: - print(f'(create_tmp_dir) could not read outdir from json_exp_config... oh well!') - - + print(f'(create_tmp_dir) could not read outdir from json_exp_config... oh well!') + + print(f"(create_tmp_dir) outdir_from_exp_config = {outdir_from_exp_config}") print(f"(create_tmp_dir) outdir = {outdir}") - assert False tmp_dir = None if any( [ outdir == "/local2", outdir.find("/work") != -1, @@ -178,9 +173,16 @@ def create_tmp_dir(outdir, json_exp_config = None): try: os.makedirs(tmp_dir, exist_ok=True) if outdir_from_exp_config is not None: - os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok=True) + print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ') + try: + os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok=True) + except: + print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work') + print( ' .... oh well! it was ust to try to avoid a warning anyways.... moving on') + pass except Exception as exc: - raise OSError('(create_tmp_dir) problem creating temp output directory. stop.') from exc + raise OSError(f'(create_tmp_dir) problem creating tmp output directory {tmp_dir}. stop.') from exc + return tmp_dir @@ -302,9 +304,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # now we set up the cmor module object # initialize CMOR cmor.setup( - netcdf_file_action = cmor.CMOR_PRESERVE, - set_verbosity = cmor.CMOR_NORMAL, #CMOR_QUIET, #default is CMOR_NORMAL - exit_control = cmor.CMOR_EXIT_ON_WARNING,#CMOR_NORMAL, + netcdf_file_action = cmor.CMOR_PRESERVE, #CMOR_APPEND,# + set_verbosity = cmor.CMOR_NORMAL, #CMOR_QUIET,# + exit_control = cmor.CMOR_NORMAL,#CMOR_EXIT_ON_WARNING,# logfile = './foo.log', create_subdirectories = 1 ) @@ -389,6 +391,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor_vert_dim_name = vert_dim if vert_dim == "landuse": cmor_vert_dim_name = "landUse" # this is why can't we have nice things + print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') cmor_lev = cmor.axis( cmor_vert_dim_name, coord_vals = lev[:], units = lev_units ) From 191aee995ea16dffc86d18ff87e9bfc23bae8019 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 19 Nov 2024 11:17:42 -0500 Subject: [PATCH 10/36] slight clean up --- fre/cmor/cmor_mixer.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 5f5f515f..175f7ac9 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -121,26 +121,21 @@ def get_vertical_dimension(ds, target_var): ''' vert_dim = 0 for name, variable in ds.variables.items(): - # not the var we are looking for? move on. - if name != target_var: - continue + if name != target_var: # not the var we are looking for? move on. + continue dims = variable.dimensions - for dim in dims: - # if it is not a vertical axis, move on. - print(f'(get_vertical_dimension) dim={dim}') - if dim == 'landuse': - #continue - print(f'(get_vertical_dimension) i think i will crash... NOW') + for dim in dims: #print(f'(get_vertical_dimension) dim={dim}') + + # check for special case + if dim == 'landuse': # aux coordinate, so has no axis property vert_dim = dim break - if dim == 'landuse': - print(f'(get_vertical_dimension) i think i will crash... NOW') + # if it is not a vertical axis, move on. if not (ds[dim].axis and ds[dim].axis == "Z"): - if dim == 'landuse': - print(f'(get_vertical_dimension) I WILL NOT SEE THIS PRINTOUT!!!') continue vert_dim = dim + return vert_dim def create_tmp_dir(outdir, json_exp_config = None): @@ -151,32 +146,36 @@ def create_tmp_dir(outdir, json_exp_config = None): file output. tmp_dir will be slightly different depending on the output directory targeted ''' + # first see if the exp_config has any additional output path structure to create outdir_from_exp_config = None if json_exp_config is not None: with open(json_exp_config, "r", encoding = "utf-8") as table_config_file: try: outdir_from_exp_config = json.load(table_config_file)["outpath"] except: - print(f'(create_tmp_dir) could not read outdir from json_exp_config... oh well!') + print(f'(create_tmp_dir) WARNING could not read outdir from json_exp_config.' + ' the cmor module will throw a toothless warning' ) - print(f"(create_tmp_dir) outdir_from_exp_config = {outdir_from_exp_config}") - print(f"(create_tmp_dir) outdir = {outdir}") + # assign an appropriate temporary working directory tmp_dir = None if any( [ outdir == "/local2", outdir.find("/work") != -1, outdir.find("/net" ) != -1 ] ): - print(f'(create_tmp_dir) using /local /work /net ( tmp_dir = {outdir}/ )') tmp_dir = str( Path("{outdir}/").resolve() ) + print(f'(create_tmp_dir) using /local /work /net ( tmp_dir = {tmp_dir} )') else: - print(f'(create_tmp_dir) NOT using /local /work /net (tmp_dir = {outdir}/tmp/ )') tmp_dir = str( Path(f"{outdir}/tmp/").resolve() ) + print(f'(create_tmp_dir) NOT using /local /work /net ( tmp_dir = {tmp_dir} )') + + # once we know where the tmp_dir should be, create it try: os.makedirs(tmp_dir, exist_ok=True) + # and if we need to additionally create outdir_from_exp_config... try doing that too if outdir_from_exp_config is not None: print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ') try: os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok=True) - except: + except: # ... but don't error out for lack of success here, not worth it. cmor can do the lift too. print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work') print( ' .... oh well! it was ust to try to avoid a warning anyways.... moving on') pass From 8608e748db1894bf6769ec23bd769b47ac77df4e Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 19 Nov 2024 16:30:26 -0500 Subject: [PATCH 11/36] correct name of original cmor_mixer author in tool-specifici doc --- docs/tools/cmor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/cmor.rst b/docs/tools/cmor.rst index ee55cf43..36caefcf 100644 --- a/docs/tools/cmor.rst +++ b/docs/tools/cmor.rst @@ -75,7 +75,7 @@ background ~~~~~~~~~~ The bulk of this routine is housed in ``fre/cmor/cmor_mixer.py``, which is a rewritten version of -Sergey Malyshev's original ``CMORcommander.py`` script, utilized during GFDL's CMIP6 publishing run. +Sergey Nikonov's original ``CMORcommander.py`` script, utilized during GFDL's CMIP6 publishing run. This code is dependent on two primary json configuration files- a MIP variable table and another containing experiment (i.e. model) specific metdata (e.g. grid) to append From c8b4dcb4362a2c1a61cbe03f036299bce1e79418 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 20 Nov 2024 17:28:16 -0500 Subject: [PATCH 12/36] add scratchwork script showing prototype land-data rewriting. small bug regarding numpy object usage- ians fault! needs from fresh eyes to find it --- run_test_file_cases.py | 32 +++++++++++- scratchwork.py | 107 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 scratchwork.py diff --git a/run_test_file_cases.py b/run_test_file_cases.py index d031c2e4..6cc4988a 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -65,6 +65,33 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return +# 9) FAIL (4 dimensional data with no vertical) +# Result - error, +# the variable gppLut needs coordinate variables time lat lon and landuse +# the landuse coordinate is expected to be of character type, and one of four different specific strings, +# each string representing primary/secondary, pasture, crops, urban style land usage. +# this file's landuse coordinate is an integer between 0 and 3, so it's not clear to the CMOR module +# how to map the integers to the string values (though it's obvious to me) +testfile_LUmip_refined_gr1_Emon_landusedim = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/LUmip_refined/ts/monthly/5yr/' + \ + 'LUmip_refined.185001-185412.gppLut.nc' +try: + some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') +if some_return != 0: + print('didnt pass the land-file test. exit.') +# sys.exit() + + + + + + # 6) FAIL (copy_nc failure!!! WEIRD) # ocean, Omon / sos # Result - error, AttributeError: NetCDF: Attempt to define fill value when data already exists. @@ -81,7 +108,7 @@ def run_cmor_RUN(filename, table, opt_var_name): print_the_outcome(some_return,'ocean_monthly_gn / sos') if some_return != 0: print('didnt pass ocean-file test number 1... exit.') - sys.exit() +# sys.exit() @@ -101,7 +128,7 @@ def run_cmor_RUN(filename, table, opt_var_name): print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') if some_return != 0: print('didnt pass ocean-file test number 2... exit.') - sys.exit() +# sys.exit() ## 1) SUCCEEDs @@ -224,3 +251,4 @@ def run_cmor_RUN(filename, table, opt_var_name): sys.exit() +kk diff --git a/scratchwork.py b/scratchwork.py new file mode 100644 index 00000000..e8133fea --- /dev/null +++ b/scratchwork.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +#import sys +#import os +from pathlib import Path + +import numpy +from netCDF4 import Dataset, stringtochar + + + +# open netcdf file in append mode? write mode? read+? +gppLut_fin=Dataset('./tmp/LUmip_refined.185001-185412.gppLut.nc', mode='r') + +gppLut_var_data = gppLut_fin.variables['gppLut'][:] +landuse_coord_data = gppLut_fin.variables['landuse'][:] +lat_coord_data = gppLut_fin.variables['lat'][:] +lon_coord_data = gppLut_fin.variables['lon'][:] +time_coord_data = gppLut_fin.variables['time'][:] + + +# we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work +# recall, a netCDF4 file is, basically, 4 sets of things +# attributes, i.e effectively global metadata +# groups, i.e. a heirarchy with nesting a lot like directories (older netcdf files only have a root group) +# dimensions, i.e. a set of named-integers to define the number of divisions on an axis +# variables, i.e. arrays representing data with shapes described by the dimensions in the file +gppLut_fout=Dataset('./tmp/PLAY_LUmip_refined.185001-185412.gppLut.nc',mode='w') + + + +# createDimension(self, dimname, size=None)... None will imply unlimited +gppLut_fout.createDimension( 'time', + None ) #gppLut_fin.dimensions['time'].size ) +gppLut_fout.createDimension( 'landUse', + 4 ) +# this will be for the landUse coordinate axis entries +# set to length of longest string in set --> i think this is OK +gppLut_fout.createDimension( 'str_len', + len("primary_and_secondary_land") ) # see below, landuse_str_list +gppLut_fout.createDimension( 'lat', + gppLut_fin.dimensions['lat'].size ) +gppLut_fout.createDimension( 'lon', + gppLut_fin.dimensions['lon'].size ) + + +#def createVariable(self, varname, datatype, dimensions=(), +# compression = None, zlib = False, complevel = 4, +# shuffle = True, szip_coding = 'nn', szip_pixels_per_block = 8, +# blosc_shuffle = 1, fletcher32 = False, contiguous = False, +# chunksizes = None, endian = 'native', least_significant_digit = None, +# significant_digits = None, quantize_mode = 'BitGroom', +# fill_value = None, chunk_cache = None ) +gppLut_fout.createVariable( 'time', gppLut_fin.variables['time'].dtype, + dimensions = (gppLut_fout.dimensions['time']) ) +gppLut_fout.variables['time'][:] = time_coord_data + +gppLut_fout.createVariable( 'lat', gppLut_fin.variables['lat'].dtype, + dimensions = (gppLut_fout.dimensions['lat']) ) +gppLut_fout.variables['lat'][:] = lat_coord_data + +gppLut_fout.createVariable( 'lon', gppLut_fin.variables['lon'].dtype, + dimensions = (gppLut_fout.dimensions['lon']) ) +gppLut_fout.variables['lon'][:] = lon_coord_data + + +# action time for a tricky coordinate variable +# in 'landuse', an int flag, values [0, 1, 2, 3] to be interpreted as: +# 0 - psl, "primary_and_secondary_land" char string in CMIP6/7 +# 1 - pst, "pastures" char string in CMIP6/7 +# 2 - crp, "crops" char string in CMIP6/7 +# 3 - urb, "urban" char string in CMIP6/7 +landuse_str_list=["primary_and_secondary_land", "pastures", "crops", "urban"] + +landUse_var = gppLut_fout.createVariable( 'landUse', 'c', #'S1' + dimensions = ( gppLut_fout.dimensions['landUse'], + gppLut_fout.dimensions['str_len'] ) ) +# fairly sure CMOR will fill this in? +#landUse_var.standard_name = "area_type" +#landUse_var.long_name = "Land use type" +#landUse_var.must_have_bounds = "no" +#landUse_var.out_name = 'landuse' + +landUse_var._Encoding = 'ascii' + +# determine datatype, dep on max string length in landuse_str_list +numpy_str_dtype = 'S' + str(len(landuse_str_list[0])) #should be 'S26' +landUse_var[:] = numpy.array(landuse_str_list, dtype=numpy_str_dtype ) + + +# data time!!! +gppLut_var_out = gppLut_fout.createVariable( 'gppLut', gppLut_fin.variables['gppLut'].dtype, + dimensions = (gppLut_fout.dimensions['time'], + gppLut_fout.dimensions['landUse'], + gppLut_fout.dimensions['lat'], + gppLut_fout.dimensions['lon'] ) ) +gppLut_var_out[:] = gppLut_var_data + +unmsk_gppLut_var_data=gppLut_var_data[~gppLut_var_data.mask] +# test that the two are equivalent "quickly"... +unmsk_gppLut_var_out=gppLut_var_out[~gppLut_var_out.mask] + +for i in range(0, len( unmsk_gppLut_var_data ) ): + if i%100 == 0: + print(f'i = {i}') + diff = unmsk_gppLut_var_data[i] - unmsk_gppLut_var_out[i] + if diff > 0.: + print(f'diff = \n {diff}') From f69b5777a634b66d4647d5af5755fa2e85893687 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 21 Nov 2024 17:20:51 -0500 Subject: [PATCH 13/36] i have a script that rewrite the land data in a compliant manner- can i get it into cmor_mixer? i can now also start tackling ocean cases, which seem at least somewhat tractable now --- fre/cmor/cmor_mixer.py | 78 ++++++++------ run_test_file_cases.py | 103 +++++++----------- scratchwork.py | 235 ++++++++++++++++++++++++++++------------- 3 files changed, 242 insertions(+), 174 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 175f7ac9..a0628f80 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -20,6 +20,7 @@ # ----- \end consts ### ------ helper functions ------ ### +import shutil def copy_nc(in_nc, out_nc): ''' copy target input netcdf file in_nc to target out_nc. I have to think this is not a trivial copy @@ -27,34 +28,40 @@ def copy_nc(in_nc, out_nc): in_nc: string, path to an input netcdf file we wish to copy out_nc: string, an output path to copy the targeted input netcdf file to ''' - print(f'(copy_nc) in_nc: {in_nc}\n' - f' out_nc: {out_nc}') + shutil.copy(in_nc, out_nc) +# print(f'(copy_nc) in_nc: {in_nc}\n' +# f' out_nc: {out_nc}') +# +# # input file +# dsin = nc.Dataset(in_nc) +# +# # output file, same exact data_model as input file. +# # note- totally infuriating... +# # the correct value for the format arg is netCDF4.Dataset.data_model +# # and NOT netCDF4.Dataset.disk_format +# dsout = nc.Dataset(out_nc, "w", +# format = dsin.data_model) +# +# #Copy dimensions +# for dname, the_dim in dsin.dimensions.items(): +# dsout.createDimension( dname, +# len(the_dim) if not the_dim.isunlimited() else None ) +# +# # Copy variables and attributes +# for v_name, varin in dsin.variables.items(): +# print(f'(copy_nc) v_name = {v_name}, datatype = {varin.datatype}, dimensions={varin.dimensions}') +# varin_dims_str_tuple = varin.dimensions +# out_var_dims_tuple = (dsout.dimensions[dim] for dim in varin_dims_str_tuple) +# #out_var = dsout.createVariable(v_name, varin.datatype, varin.dimensions) +# out_var = dsout.createVariable(v_name, varin.datatype, out_var_dims_tuple) +# out_var.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()}) +# out_var[:] = varin[:] +# dsout.setncatts({a:dsin.getncattr(a) for a in dsin.ncattrs()}) +# +# # close up +# dsin.close() +# dsout.close() - # input file - dsin = nc.Dataset(in_nc) - - # output file, same exact data_model as input file. - # note- totally infuriating... - # the correct value for the format arg is netCDF4.Dataset.data_model - # and NOT netCDF4.Dataset.disk_format - dsout = nc.Dataset(out_nc, "w", - format = dsin.data_model) - - #Copy dimensions - for dname, the_dim in dsin.dimensions.items(): - dsout.createDimension( dname, - len(the_dim) if not the_dim.isunlimited() else None ) - - # Copy variables and attributes - for v_name, varin in dsin.variables.items(): - out_var = dsout.createVariable(v_name, varin.datatype, varin.dimensions) - out_var.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()}) - out_var[:] = varin[:] - dsout.setncatts({a:dsin.getncattr(a) for a in dsin.ncattrs()}) - - # close up - dsin.close() - dsout.close() def get_var_filenames(indir, var_filenames = None, local_var = None): @@ -108,8 +115,9 @@ def check_dataset_for_ocean_grid(ds): ds: netCDF4.Dataset object containing variables with associated dimensional information. ''' if "xh" in list(ds.variables.keys()): - raise NotImplementedError( - "(check_dataset_for_ocean_grid) 'xh' found in var_list. ocean grid req'd but not yet unimplemented. stop.") + pass + #raise NotImplementedError( + print( "(check_dataset_for_ocean_grid) 'xh' found in var_list. ocean grid req'd but not yet unimplemented. stop.") def get_vertical_dimension(ds, target_var): @@ -127,7 +135,7 @@ def get_vertical_dimension(ds, target_var): for dim in dims: #print(f'(get_vertical_dimension) dim={dim}') # check for special case - if dim == 'landuse': # aux coordinate, so has no axis property + if dim.lower() == 'landuse': # aux coordinate, so has no axis property vert_dim = dim break @@ -293,11 +301,11 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # error if vert_dim wrong given var_dim lev, lev_units = None, "1" #1 #"none" #None #"" if vert_dim != 0: - if vert_dim not in [ "landuse", "plev39", "plev30", "plev19", "plev8", + if vert_dim.lower() not in [ "landuse", "plev39", "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') lev = ds[vert_dim] - if vert_dim != "landuse": + if vert_dim.lower() != "landuse": lev_units = ds[vert_dim].units # now we set up the cmor module object @@ -306,7 +314,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, netcdf_file_action = cmor.CMOR_PRESERVE, #CMOR_APPEND,# set_verbosity = cmor.CMOR_NORMAL, #CMOR_QUIET,# exit_control = cmor.CMOR_NORMAL,#CMOR_EXIT_ON_WARNING,# - logfile = './foo.log', +# logfile = './foo.log', create_subdirectories = 1 ) @@ -385,10 +393,10 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor_lev = None if lev is not None: print(f'(rewrite_netcdf_file_var) assigning cmor_lev') - if vert_dim in ["landuse", "plev39", "plev30", "plev19", "plev8", "height2m"]: + if vert_dim.lower() in ["landuse", "plev39", "plev30", "plev19", "plev8", "height2m"]: print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') cmor_vert_dim_name = vert_dim - if vert_dim == "landuse": + if vert_dim.lower() == "landuse": cmor_vert_dim_name = "landUse" # this is why can't we have nice things print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') cmor_lev = cmor.axis( cmor_vert_dim_name, diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 6cc4988a..cd4006d8 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -21,7 +21,7 @@ def print_the_outcome(some_return,case_str): print(f'{case_str} case probably OK [[[PROB-OK ^-^]]]: some_return={some_return}') print('-----------------------------------------------------------------------------------------------------------------') print(f'\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n') - assert some_return == 0 +# assert some_return == 0 # global consts for these tests, with no/trivial impact on the results ROOTDIR='fre/tests/test_files' @@ -65,50 +65,51 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return -# 9) FAIL (4 dimensional data with no vertical) -# Result - error, -# the variable gppLut needs coordinate variables time lat lon and landuse -# the landuse coordinate is expected to be of character type, and one of four different specific strings, -# each string representing primary/secondary, pasture, crops, urban style land usage. -# this file's landuse coordinate is an integer between 0 and 3, so it's not clear to the CMOR module -# how to map the integers to the string values (though it's obvious to me) -testfile_LUmip_refined_gr1_Emon_landusedim = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/LUmip_refined/ts/monthly/5yr/' + \ - 'LUmip_refined.185001-185412.gppLut.nc' -try: - some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') -if some_return != 0: - print('didnt pass the land-file test. exit.') -# sys.exit() +#### THIS CASE MAY WORK if i rewrite the land file correctly +## 9) FAIL (4 dimensional data with no vertical) +## Result - error, +## the variable gppLut needs coordinate variables time lat lon and landuse +## the landuse coordinate is expected to be of character type, and one of four different specific strings, +## each string representing primary/secondary, pasture, crops, urban style land usage. +## this file's landuse coordinate is an integer between 0 and 3, so it's not clear to the CMOR module +## how to map the integers to the string values (though it's obvious to me) +#testfile_LUmip_refined_gr1_Emon_landusedim = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' +#try: +# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') +#if some_return != 0: +# print('didnt pass the land-file test. exit.') +# #sys.exit() -# 6) FAIL (copy_nc failure!!! WEIRD) -# ocean, Omon / sos -# Result - error, AttributeError: NetCDF: Attempt to define fill value when data already exists. -testfile_ocean_monthly_gn = \ - '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/ocean_monthly/ts/monthly/5yr/' + \ - 'ocean_monthly.002101-002512.sos.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'ocean_monthly_gn / sos') -if some_return != 0: - print('didnt pass ocean-file test number 1... exit.') -# sys.exit() +## 6) FAIL (copy_nc failure!!! WEIRD) +## ocean, Omon / sos +## Result - error, AttributeError: NetCDF: Attempt to define fill value when data already exists. +#testfile_ocean_monthly_gn = \ +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly/ts/monthly/5yr/' + \ +# 'ocean_monthly.002101-002512.sos.nc' +#try: +# some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'ocean_monthly_gn / sos') +#if some_return != 0: +# print('didnt pass ocean-file test number 1... exit.') +## sys.exit() @@ -228,27 +229,3 @@ def run_cmor_RUN(filename, table, opt_var_name): -# 9) FAIL (4 dimensional data with no vertical) -# Result - error, -# the variable gppLut needs coordinate variables time lat lon and landuse -# the landuse coordinate is expected to be of character type, and one of four different specific strings, -# each string representing primary/secondary, pasture, crops, urban style land usage. -# this file's landuse coordinate is an integer between 0 and 3, so it's not clear to the CMOR module -# how to map the integers to the string values (though it's obvious to me) -testfile_LUmip_refined_gr1_Emon_landusedim = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/LUmip_refined/ts/monthly/5yr/' + \ - 'LUmip_refined.185001-185412.gppLut.nc' -try: - some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') -if some_return != 0: - print('didnt pass the land-file test. exit.') - sys.exit() - - -kk diff --git a/scratchwork.py b/scratchwork.py index e8133fea..450d0a9a 100644 --- a/scratchwork.py +++ b/scratchwork.py @@ -4,104 +4,187 @@ from pathlib import Path import numpy -from netCDF4 import Dataset, stringtochar +#from numpy.dtypes import StringDType +from netCDF4 import Dataset #, stringtochar +''' + in landuse, an int flag, values [0, 1, 2, 3] to be interpreted as: + 0 - psl, "primary_and_secondary_land" char string in CMIP6/7 + 1 - pst, "pastures" char string in CMIP6/7 + 2 - crp, "crops" char string in CMIP6/7 + 3 - urb, "urban" char string in CMIP6/7 +''' +#landuse_str_list=["primary_and_secondary_land", "pastures", "crops", "urban"] +landuse_str_list=[b"primary_and_secondary_land", b"pastures", b"crops", b"urban"] # open netcdf file in append mode? write mode? read+? gppLut_fin=Dataset('./tmp/LUmip_refined.185001-185412.gppLut.nc', mode='r') +gppLut_fin_ncattrs=gppLut_fin.__dict__ #dictionary -gppLut_var_data = gppLut_fin.variables['gppLut'][:] -landuse_coord_data = gppLut_fin.variables['landuse'][:] -lat_coord_data = gppLut_fin.variables['lat'][:] -lon_coord_data = gppLut_fin.variables['lon'][:] -time_coord_data = gppLut_fin.variables['time'][:] - +# the target data of interest +gppLut_var_data = gppLut_fin.variables['gppLut'][:] +gppLut_var_atts = gppLut_fin.variables['gppLut'].__dict__ -# we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work -# recall, a netCDF4 file is, basically, 4 sets of things -# attributes, i.e effectively global metadata -# groups, i.e. a heirarchy with nesting a lot like directories (older netcdf files only have a root group) -# dimensions, i.e. a set of named-integers to define the number of divisions on an axis -# variables, i.e. arrays representing data with shapes described by the dimensions in the file -gppLut_fout=Dataset('./tmp/PLAY_LUmip_refined.185001-185412.gppLut.nc',mode='w') +# coordinate variables, their _bnds, and their identically named dimensions +bnds_coord_data = gppLut_fin.variables['bnds'][:] +bnds_coord_atts = gppLut_fin.variables['bnds'].__dict__ +bnds_coord_dims = gppLut_fin.dimensions['bnds'].size +time_coord_data = gppLut_fin.variables['time'][:] +time_coord_atts = gppLut_fin.variables['time'].__dict__ +time_coord_bnds = gppLut_fin.variables['time_bnds'][:] +time_coord_bnds_atts = gppLut_fin.variables['time_bnds'].__dict__ +#time_coord_dims = gppLut_fin.dimensions['time'].size - -# createDimension(self, dimname, size=None)... None will imply unlimited +landuse_coord_data = gppLut_fin.variables['landuse'][:] +landuse_coord_atts = gppLut_fin.variables['landuse'].__dict__ +landuse_coord_dims = gppLut_fin.dimensions['landuse'].size + +lat_coord_data = gppLut_fin.variables['lat'][:] +lat_coord_atts = gppLut_fin.variables['lat'].__dict__ +lat_coord_bnds = gppLut_fin.variables['lat_bnds'][:] +lat_coord_bnds_atts = gppLut_fin.variables['lat_bnds'].__dict__ +lat_coord_dims = gppLut_fin.dimensions['lat'].size + +lon_coord_data = gppLut_fin.variables['lon'][:] +lon_coord_atts = gppLut_fin.variables['lon'].__dict__ +lon_coord_bnds = gppLut_fin.variables['lon_bnds'][:] +lon_coord_bnds_atts = gppLut_fin.variables['lon_bnds'].__dict__ +lon_coord_dims = gppLut_fin.dimensions['lon'].size + +''' + we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work + recall, a netCDF4 file is, basically, 4 sets of things + attributes, i.e effectively global metadata + groups, i.e. a heirarchy with nesting a lot like directories (older netcdf files only have a root group) + dimensions, i.e. a set of named-integers to define the number of divisions on an axis + variables, i.e. arrays representing data with shapes described by the dimensions in the file +''' + +# open the output file +gppLut_fout=Dataset('./alt_LUmip_input/PLAY_LUmip_refined.185001-185412.gppLut.nc',mode='w') +gppLut_fout.setncatts(gppLut_fin_ncattrs) + +''' + from netCDF4 python API doc, for easy referencing + createDimension(self, + dimname, size=None)... None will imply unlimited +''' gppLut_fout.createDimension( 'time', - None ) #gppLut_fin.dimensions['time'].size ) -gppLut_fout.createDimension( 'landUse', - 4 ) + None ) #time_coord_dims +gppLut_fout.createDimension( 'bnds', + bnds_coord_dims ) # this will be for the landUse coordinate axis entries +gppLut_fout.createDimension( 'landUse', + landuse_coord_dims ) # set to length of longest string in set --> i think this is OK gppLut_fout.createDimension( 'str_len', - len("primary_and_secondary_land") ) # see below, landuse_str_list + len("primary_and_secondary_land") ) # see below, landuse_str_list gppLut_fout.createDimension( 'lat', - gppLut_fin.dimensions['lat'].size ) + lat_coord_dims ) gppLut_fout.createDimension( 'lon', - gppLut_fin.dimensions['lon'].size ) - - -#def createVariable(self, varname, datatype, dimensions=(), -# compression = None, zlib = False, complevel = 4, -# shuffle = True, szip_coding = 'nn', szip_pixels_per_block = 8, -# blosc_shuffle = 1, fletcher32 = False, contiguous = False, -# chunksizes = None, endian = 'native', least_significant_digit = None, -# significant_digits = None, quantize_mode = 'BitGroom', -# fill_value = None, chunk_cache = None ) -gppLut_fout.createVariable( 'time', gppLut_fin.variables['time'].dtype, - dimensions = (gppLut_fout.dimensions['time']) ) -gppLut_fout.variables['time'][:] = time_coord_data + lon_coord_dims ) -gppLut_fout.createVariable( 'lat', gppLut_fin.variables['lat'].dtype, - dimensions = (gppLut_fout.dimensions['lat']) ) -gppLut_fout.variables['lat'][:] = lat_coord_data -gppLut_fout.createVariable( 'lon', gppLut_fin.variables['lon'].dtype, - dimensions = (gppLut_fout.dimensions['lon']) ) -gppLut_fout.variables['lon'][:] = lon_coord_data -# action time for a tricky coordinate variable -# in 'landuse', an int flag, values [0, 1, 2, 3] to be interpreted as: -# 0 - psl, "primary_and_secondary_land" char string in CMIP6/7 -# 1 - pst, "pastures" char string in CMIP6/7 -# 2 - crp, "crops" char string in CMIP6/7 -# 3 - urb, "urban" char string in CMIP6/7 -landuse_str_list=["primary_and_secondary_land", "pastures", "crops", "urban"] +''' + from netCDF4 python API doc, for easy referencing + def createVariable(self, + varname, datatype, dimensions=(), + lots others ) +''' +# easy variables first. +# bnds +gppLut_fout.createVariable( 'bnds', gppLut_fin.variables['bnds'].dtype, + dimensions = ( gppLut_fout.dimensions['bnds'] ) ) +gppLut_fout.variables['bnds'][:] = bnds_coord_data +gppLut_fout.variables['bnds'].setncatts( bnds_coord_atts ) -landUse_var = gppLut_fout.createVariable( 'landUse', 'c', #'S1' - dimensions = ( gppLut_fout.dimensions['landUse'], - gppLut_fout.dimensions['str_len'] ) ) -# fairly sure CMOR will fill this in? -#landUse_var.standard_name = "area_type" -#landUse_var.long_name = "Land use type" -#landUse_var.must_have_bounds = "no" -#landUse_var.out_name = 'landuse' +# time +gppLut_fout.createVariable( 'time', gppLut_fin.variables['time'].dtype, + dimensions = ( gppLut_fout.dimensions['time'] ) ) +gppLut_fout.variables['time'][:] = time_coord_data +gppLut_fout.variables['time'].setncatts( time_coord_atts ) + +# time_bnds +gppLut_fout.createVariable( 'time_bnds', gppLut_fin.variables['time_bnds'].dtype, + fill_value = gppLut_fin.variables['time_bnds']._FillValue, # necessary bc of unlimited + extra limited dim shape? + dimensions = ( gppLut_fout.dimensions['time'], + gppLut_fout.dimensions['bnds'] ) ) +gppLut_fout.variables['time_bnds'][:] = time_coord_bnds +for att in time_coord_bnds_atts: #gppLut_fout.variables['time_bnds'].setncatts( time_coord_bnds_atts ) + if att != '_FillValue': + gppLut_fout.variables['time_bnds'].setncattr( att, time_coord_bnds_atts[att] ) + +# lat +gppLut_fout.createVariable( 'lat', gppLut_fin.variables['lat'].dtype, + dimensions = ( gppLut_fout.dimensions['lat'] ) ) +gppLut_fout.variables['lat'][:] = lat_coord_data +gppLut_fout.variables['lat'].setncatts( lat_coord_atts ) + +# lat_bnds +gppLut_fout.createVariable( 'lat_bnds', gppLut_fin.variables['lat_bnds'].dtype, + dimensions = ( gppLut_fout.dimensions['lat'], + gppLut_fout.dimensions['bnds'] ) ) +gppLut_fout.variables['lat_bnds'][:] = lat_coord_bnds +gppLut_fout.variables['lat_bnds'].setncatts( lat_coord_bnds_atts ) + +# lon +gppLut_fout.createVariable( 'lon', gppLut_fin.variables['lon'].dtype, + dimensions = ( gppLut_fout.dimensions['lon'] ) ) +gppLut_fout.variables['lon'][:] = lon_coord_data +gppLut_fout.variables['lon'].setncatts( lon_coord_atts ) + +# lon_bnds +gppLut_fout.createVariable( 'lon_bnds', gppLut_fin.variables['lon_bnds'].dtype, + dimensions = ( gppLut_fout.dimensions['lon'], + gppLut_fout.dimensions['bnds'] ) ) +gppLut_fout.variables['lon_bnds'][:] = lon_coord_bnds +gppLut_fout.variables['lon_bnds'].setncatts( lon_coord_bnds_atts ) + +# landUse +gppLut_fout.createVariable( 'landUse', 'c', #'S1' + dimensions = ( gppLut_fout.dimensions['landUse'], + gppLut_fout.dimensions['str_len'] ) ) +gppLut_fout.variables['landUse'].setncattr( '_Encoding', 'ascii' ) +# determine numpy array datatype, dep on max string length in landuse_str_list +#landUse_datatype = numpy.dtype([ +gppLut_fout.variables['landUse'][:] = numpy.array(landuse_str_list, dtype= f'S{len(landuse_str_list[0])}')#)# +#landUse_var[:] = numpy.array(landuse_str_list, dtype= f'S{len(landuse_str_list[0])}') -landUse_var._Encoding = 'ascii' +# fairly sure CMOR will fill this in? +gppLut_fout.variables['landUse'].setncattr( + 'requested_values', ' '.join( + [ landuse_str.decode() for landuse_str in landuse_str_list ] ) + ) +gppLut_fout.variables['landUse'].setncattr("standard_name" , "area_type" ) +gppLut_fout.variables['landUse'].setncattr("long_name" , "Land use type" ) +gppLut_fout.variables['landUse'].setncattr("must_have_bounds" , "no" ) +gppLut_fout.variables['landUse'].setncattr("out_name" , 'landuse' ) -# determine datatype, dep on max string length in landuse_str_list -numpy_str_dtype = 'S' + str(len(landuse_str_list[0])) #should be 'S26' -landUse_var[:] = numpy.array(landuse_str_list, dtype=numpy_str_dtype ) # data time!!! -gppLut_var_out = gppLut_fout.createVariable( 'gppLut', gppLut_fin.variables['gppLut'].dtype, - dimensions = (gppLut_fout.dimensions['time'], - gppLut_fout.dimensions['landUse'], - gppLut_fout.dimensions['lat'], - gppLut_fout.dimensions['lon'] ) ) -gppLut_var_out[:] = gppLut_var_data - -unmsk_gppLut_var_data=gppLut_var_data[~gppLut_var_data.mask] -# test that the two are equivalent "quickly"... -unmsk_gppLut_var_out=gppLut_var_out[~gppLut_var_out.mask] - -for i in range(0, len( unmsk_gppLut_var_data ) ): - if i%100 == 0: - print(f'i = {i}') - diff = unmsk_gppLut_var_data[i] - unmsk_gppLut_var_out[i] - if diff > 0.: - print(f'diff = \n {diff}') +gppLut_fout.createVariable( 'gppLut', gppLut_fin.variables['gppLut'].dtype, + fill_value = gppLut_fin.variables['gppLut']._FillValue, + dimensions = ( gppLut_fout.dimensions['time'], + gppLut_fout.dimensions['landUse'], + gppLut_fout.dimensions['lat'], + gppLut_fout.dimensions['lon'] ) ) +gppLut_fout.variables['gppLut'][:] = gppLut_var_data +for att in gppLut_var_atts: + if att not in ["time_avg_info", "_FillValue"]: + gppLut_fout.variables['gppLut'].setncattr(att, gppLut_var_atts[att] ) + +gppLut_fout.close() + +## test that the two are equivalent "quickly"... +#unmsk_gppLut_var_data=gppLut_var_data[~gppLut_var_data.mask] +#unmsk_gppLut_var_out=gppLut_var_out[~gppLut_var_out.mask] +#for i in range(0, len( unmsk_gppLut_var_data ) ): +# if i%100 == 0: +# print(f'i = {i}') +# diff = unmsk_gppLut_var_data[i] - unmsk_gppLut_var_out[i] +# if diff > 0.: +# print(f'diff = \n {diff}') From 0b769dabf9cfd46c9aeb0f34b21611cbd182c72a Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 3 Dec 2024 15:17:37 -0500 Subject: [PATCH 14/36] poke the test file cases script --- run_test_file_cases.py | 48 +++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/run_test_file_cases.py b/run_test_file_cases.py index cd4006d8..dd6c0fd1 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -5,7 +5,6 @@ it is for a very context-dependent set of tests for a very specific point in time. ''' - import sys import os from pathlib import Path @@ -16,12 +15,12 @@ def print_the_outcome(some_return,case_str): print('-----------------------------------------------------------------------------------------------------------------') if some_return != 0: - print(f'{case_str} case failed[[[FAIL -_-]]]: some_return={some_return}') + print(f'{case_str} case failed [[[ FAIL -_- ]]]: some_return={some_return}') else: - print(f'{case_str} case probably OK [[[PROB-OK ^-^]]]: some_return={some_return}') + print(f'{case_str} case probably OK [[[ PROB-OK ^-^ ]]]: some_return={some_return}') print('-----------------------------------------------------------------------------------------------------------------') print(f'\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n') -# assert some_return == 0 + #assert some_return == 0 # global consts for these tests, with no/trivial impact on the results ROOTDIR='fre/tests/test_files' @@ -90,32 +89,29 @@ def run_cmor_RUN(filename, table, opt_var_name): +# 6) FAIL +# ocean, Omon / sos +# Result - error, +testfile_ocean_monthly_gn = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly/ts/monthly/5yr/' + \ + 'ocean_monthly.002101-002512.sos.nc' +try: + some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'ocean_monthly_gn / sos') +if some_return != 0: + print('didnt pass ocean-file test number 1... exit.') + sys.exit() -## 6) FAIL (copy_nc failure!!! WEIRD) -## ocean, Omon / sos -## Result - error, AttributeError: NetCDF: Attempt to define fill value when data already exists. -#testfile_ocean_monthly_gn = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/ocean_monthly/ts/monthly/5yr/' + \ -# 'ocean_monthly.002101-002512.sos.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'ocean_monthly_gn / sos') -#if some_return != 0: -# print('didnt pass ocean-file test number 1... exit.') -## sys.exit() - - - -# 7) FAIL (copy_nc failure!!! WEIRD) +# 7) FAIL # ocean, Omon / so -# Result - identical failure to #6 +# Result - error, testfile_ocean_monthly_z_1x1deg_gr = \ '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ From 2d686497420778fcc6f1d09b46cb4f568da1a149 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 3 Dec 2024 15:50:02 -0500 Subject: [PATCH 15/36] adjust comment helpfulling re coord variable --- scratchwork.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scratchwork.py b/scratchwork.py index 450d0a9a..8abfaa8d 100644 --- a/scratchwork.py +++ b/scratchwork.py @@ -27,6 +27,8 @@ gppLut_var_atts = gppLut_fin.variables['gppLut'].__dict__ # coordinate variables, their _bnds, and their identically named dimensions +# coordinate variable == a variable with the same name as a dimension. +# pitfall: an "axis" in netcdf is not analagous to a dimension, overloaded term bnds_coord_data = gppLut_fin.variables['bnds'][:] bnds_coord_atts = gppLut_fin.variables['bnds'].__dict__ bnds_coord_dims = gppLut_fin.dimensions['bnds'].size From e7933c0c65220cdcb8fd86d00f4c9f3187d42cfb Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 3 Dec 2024 15:50:42 -0500 Subject: [PATCH 16/36] print cwd between tests, failures in cmor during its bulk routine may leave the python interpreter instance in the wrong directory --- run_test_file_cases.py | 55 ++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/run_test_file_cases.py b/run_test_file_cases.py index dd6c0fd1..e8fdfb17 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -12,6 +12,11 @@ import fre from fre.cmor.cmor_mixer import cmor_run_subtool as run_cmor + +def print_cwd(): + print(f'os.getcwd() = {os.getcwd()}') + print(f'\n\n\n\n') + def print_the_outcome(some_return,case_str): print('-----------------------------------------------------------------------------------------------------------------') if some_return != 0: @@ -19,7 +24,8 @@ def print_the_outcome(some_return,case_str): else: print(f'{case_str} case probably OK [[[ PROB-OK ^-^ ]]]: some_return={some_return}') print('-----------------------------------------------------------------------------------------------------------------') - print(f'\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n') + print(f'\n\n\n\n\n\n\n\n\n\n') + print_cwd() #assert some_return == 0 # global consts for these tests, with no/trivial impact on the results @@ -64,28 +70,24 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return -#### THIS CASE MAY WORK if i rewrite the land file correctly -## 9) FAIL (4 dimensional data with no vertical) -## Result - error, -## the variable gppLut needs coordinate variables time lat lon and landuse -## the landuse coordinate is expected to be of character type, and one of four different specific strings, -## each string representing primary/secondary, pasture, crops, urban style land usage. -## this file's landuse coordinate is an integer between 0 and 3, so it's not clear to the CMOR module -## how to map the integers to the string values (though it's obvious to me) -#testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/LUmip_refined/ts/monthly/5yr/' + \ -# 'LUmip_refined.185001-185412.gppLut.nc' -#try: -# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_langusedim / gppLut') -#if some_return != 0: -# print('didnt pass the land-file test. exit.') -# #sys.exit() +### THIS CASE MAY WORK if i rewrite the land file correctly +# 9) FAIL (4 dimensional data with no vertical) +# Result - type error, expecting landuse dimension to be string, not an int flag with interpretation +# Fix - rewrite the file and it's data, mapping it's landuse int flag coord to the desired string values +testfile_LUmip_refined_gr1_Emon_landusedim = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/LUmip_refined/ts/monthly/5yr/' + \ + 'LUmip_refined.185001-185412.gppLut.nc' +try: + some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') +if some_return != 0: + print('didnt pass the land-file test. exit.') + #sys.exit() @@ -105,7 +107,7 @@ def run_cmor_RUN(filename, table, opt_var_name): print_the_outcome(some_return,'ocean_monthly_gn / sos') if some_return != 0: print('didnt pass ocean-file test number 1... exit.') - sys.exit() +# sys.exit() @@ -128,6 +130,8 @@ def run_cmor_RUN(filename, table, opt_var_name): # sys.exit() + + ## 1) SUCCEEDs ## land, Lmon, gr1 #testfile_land_gr1_Lmon = \ @@ -225,3 +229,6 @@ def run_cmor_RUN(filename, table, opt_var_name): + + + From a5dd1e7edff89c5db7eb48b670f822a504dee157 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 3 Dec 2024 15:51:23 -0500 Subject: [PATCH 17/36] add try-except-finally behavior to calling rewrite_netcdf_file_var, so the cwd for the os gets set back to what it was before fre cmor run was called --- fre/cmor/cmor_mixer.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index a0628f80..28c955c5 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -584,19 +584,24 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None, gotta_go_back_here=os.getcwd() try: - print(f"cd'ing to \n {make_cmor_write_here}" ) + print(f"(cmorize_target_var_files) WARNING changing directory to: \n {make_cmor_write_here}" ) os.chdir( make_cmor_write_here ) except: raise OSError(f'could not chdir to {make_cmor_write_here}') print ("(cmorize_target_var_files) calling rewrite_netcdf_file_var") - local_file_name = rewrite_netcdf_file_var( proj_table_vars , - local_var , - nc_file_work , - target_var , - json_exp_config , - json_table_config ) - os.chdir( gotta_go_back_here ) + try: + local_file_name = rewrite_netcdf_file_var( proj_table_vars , + local_var , + nc_file_work , + target_var , + json_exp_config , + json_table_config ) + except: + raise Exception('(cmorize_target_var_files) problem with rewrite_netcdf_file_var. exiting and executing finally block.') + finally: + print(f'(cmorize_target_var_files) WARNING changing directory to: \n {gotta_go_back_here}') + os.chdir( gotta_go_back_here ) # now that CMOR has rewritten things... we can take our post-rewriting actions From 2681272f1776db1d51dc4fec1fd2d9a8087bb1d1 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 3 Dec 2024 16:01:56 -0500 Subject: [PATCH 18/36] formally remove unneeded copy_nc function. replace iwth shutil. better exception reporting for try/except/finally call of rewrite_netcdf_file_var --- fre/cmor/cmor_mixer.py | 58 +++++++----------------------------------- 1 file changed, 9 insertions(+), 49 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 28c955c5..881a2879 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -7,6 +7,7 @@ import os import glob import json +import shutil import subprocess from pathlib import Path @@ -16,53 +17,10 @@ # ----- \start consts DEBUG_MODE_RUN_ONE = True - # ----- \end consts -### ------ helper functions ------ ### -import shutil -def copy_nc(in_nc, out_nc): - ''' - copy target input netcdf file in_nc to target out_nc. I have to think this is not a trivial copy - operation, as if it were, using shutil's copy would be sufficient. accepts two arguments - in_nc: string, path to an input netcdf file we wish to copy - out_nc: string, an output path to copy the targeted input netcdf file to - ''' - shutil.copy(in_nc, out_nc) -# print(f'(copy_nc) in_nc: {in_nc}\n' -# f' out_nc: {out_nc}') -# -# # input file -# dsin = nc.Dataset(in_nc) -# -# # output file, same exact data_model as input file. -# # note- totally infuriating... -# # the correct value for the format arg is netCDF4.Dataset.data_model -# # and NOT netCDF4.Dataset.disk_format -# dsout = nc.Dataset(out_nc, "w", -# format = dsin.data_model) -# -# #Copy dimensions -# for dname, the_dim in dsin.dimensions.items(): -# dsout.createDimension( dname, -# len(the_dim) if not the_dim.isunlimited() else None ) -# -# # Copy variables and attributes -# for v_name, varin in dsin.variables.items(): -# print(f'(copy_nc) v_name = {v_name}, datatype = {varin.datatype}, dimensions={varin.dimensions}') -# varin_dims_str_tuple = varin.dimensions -# out_var_dims_tuple = (dsout.dimensions[dim] for dim in varin_dims_str_tuple) -# #out_var = dsout.createVariable(v_name, varin.datatype, varin.dimensions) -# out_var = dsout.createVariable(v_name, varin.datatype, out_var_dims_tuple) -# out_var.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()}) -# out_var[:] = varin[:] -# dsout.setncatts({a:dsin.getncattr(a) for a in dsin.ncattrs()}) -# -# # close up -# dsin.close() -# dsout.close() - +#### ------ helper functions ------ ### def get_var_filenames(indir, var_filenames = None, local_var = None): ''' @@ -559,14 +517,14 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None, nc_file_work = f"{tmp_dir}{name_of_set}.{iso_datetime}.{local_var}.nc" print(f"(cmorize_target_var_files) nc_file_work = {nc_file_work}") - copy_nc( nc_fls[i], nc_file_work) + shutil.copy(nc_fls[i], nc_file_work) # if the ps file exists, we'll copy it to the work directory too nc_ps_file = nc_fls[i].replace(f'.{local_var}.nc', '.ps.nc') nc_ps_file_work = nc_file_work.replace(f'.{local_var}.nc', '.ps.nc') if Path(nc_ps_file).exists(): print(f"(cmorize_target_var_files) nc_ps_file_work = {nc_ps_file_work}") - copy_nc(nc_ps_file, nc_ps_file_work) + shutil.copy(nc_ps_file, nc_ps_file_work) # TODO think of better way to write this kind of conditional data movement... @@ -597,9 +555,11 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None, target_var , json_exp_config , json_table_config ) - except: - raise Exception('(cmorize_target_var_files) problem with rewrite_netcdf_file_var. exiting and executing finally block.') - finally: + except Exception as exc: + raise Exception('(cmorize_target_var_files) problem with rewrite_netcdf_file_var. exc=\n' + f' {exc}\n' + ' exiting and executing finally block.') + finally: # should always execute, errors or not! print(f'(cmorize_target_var_files) WARNING changing directory to: \n {gotta_go_back_here}') os.chdir( gotta_go_back_here ) From ad73044d57533e3ec93fffc48c34d9752d6ae055 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 3 Dec 2024 17:48:38 -0500 Subject: [PATCH 19/36] new subtool routine, fre cmor list. --- fre/cmor/__init__.py | 1 + fre/cmor/cmor_mixer.py | 6 ++-- fre/cmor/frecmor.py | 60 +++++++++++++++++++++++++++++----- fre/tests/test_fre_cmor_cli.py | 17 ++++++++++ 4 files changed, 73 insertions(+), 11 deletions(-) diff --git a/fre/cmor/__init__.py b/fre/cmor/__init__.py index 465ad02c..eb84d10a 100644 --- a/fre/cmor/__init__.py +++ b/fre/cmor/__init__.py @@ -1,2 +1,3 @@ ''' for fre.cmor imports ''' from .cmor_mixer import cmor_run_subtool +from .cmor_lister import cmor_list_subtool diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 881a2879..4f511a81 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -269,9 +269,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # now we set up the cmor module object # initialize CMOR cmor.setup( - netcdf_file_action = cmor.CMOR_PRESERVE, #CMOR_APPEND,# - set_verbosity = cmor.CMOR_NORMAL, #CMOR_QUIET,# - exit_control = cmor.CMOR_NORMAL,#CMOR_EXIT_ON_WARNING,# + netcdf_file_action = cmor.CMOR_PRESERVE, #.CMOR_APPEND,# + set_verbosity = cmor.CMOR_QUIET,#.CMOR_NORMAL, # + exit_control = cmor.CMOR_NORMAL,#.CMOR_EXIT_ON_WARNING,# # logfile = './foo.log', create_subdirectories = 1 ) diff --git a/fre/cmor/frecmor.py b/fre/cmor/frecmor.py index e882186a..bc482ff3 100644 --- a/fre/cmor/frecmor.py +++ b/fre/cmor/frecmor.py @@ -2,8 +2,17 @@ import click +from .cmor_lister import _cmor_list_subtool from .cmor_mixer import _cmor_run_subtool +OPT_VAR_NAME_HELP="optional, specify a variable name to specifically process only filenames " + \ + "matching that variable name. I.e., this string help target local_vars, not " + \ + "target_vars." +VARLIST_HELP="path pointing to a json file containing directory of key/value pairs. " + \ + "the keys are the \'local\' names used in the filename, and the values " + \ + "pointed to by those keys are strings representing the name of the variable " + \ + "contained in targeted files. the key and value are often the same, " + \ + "but it is not required." @click.group(help=click.style(" - access fre cmor subcommands", fg=(232,91,204))) def cmor_cli(): ''' entry point to fre cmor click commands ''' @@ -16,11 +25,7 @@ def cmor_cli(): required=True) @click.option("-l", "--varlist", type=str, - help="path pointing to a json file containing directory of key/value pairs. " + \ - "the keys are the \'local\' names used in the filename, and the values " + \ - "pointed to by those keys are strings representing the name of the variable " + \ - "contained in targeted files. the key and value are often the same, " + \ - "but it is not required.", + help=VARLIST_HELP, required=True) @click.option("-r", "--table_config", type=str, @@ -40,9 +45,7 @@ def cmor_cli(): required=True) @click.option('-v', "--opt_var_name", type = str, - help="optional, specify a variable name to specifically process only filenames " + \ - "matching that variable name. I.e., this string help target local_vars, not " + \ - "target_vars.", + help=OPT_VAR_NAME_HELP, required=False) @click.pass_context def run(context, indir, varlist, table_config, exp_config, outdir, opt_var_name): @@ -62,5 +65,46 @@ def run(context, indir, varlist, table_config, exp_config, outdir, opt_var_name) # context.forward( # _cmor_run_subtool() ) + +@cmor_cli.command() +@click.option("-l", "--varlist", + type=str, + help=VARLIST_HELP, + required=False) +@click.option("-r", "--table_config_dir", + type=str, + help="directory holding MIP tables to search for variables in var list", + required=True) +@click.option('-v', "--opt_var_name", + type = str, + help=OPT_VAR_NAME_HELP, + required=False) +@click.pass_context +def list(context, varlist, table_config_dir, opt_var_name): + ''' + loop over json table files in config_dir and show which tables contain variables in var list/ + the tool will also print what that table entry is expecting of that variable as well. if given + an opt_var_name in addition to varlist, only that variable name will be printed out. + accepts 3 arguments, two of the three required. + ''' + #context.forward( _cmor_list_subtool ) + + # if opt_var_name specified, forget the list. + if opt_var_name is not None: + varlist=None + + # custom arg requirement of "one of the two or both" in click should be implemented with + # logic before calling context.invoke( , *args ) + if opt_var_name is None and varlist is None: + raise ValueError('opt_var_name and varlist cannot both be None') + + context.invoke( + _cmor_list_subtool, + json_var_list = varlist, + json_table_config_dir = table_config_dir, + opt_var_name = opt_var_name + ) + + if __name__ == "__main__": cmor_cli() diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index b2a09b62..cbc1e3ec 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -132,3 +132,20 @@ def test_cli_fre_cmor_run_case2(): assert all ( [ result.exit_code == 0, Path(full_outputfile).exists(), Path(full_inputfile).exists() ] ) + + +# fre cmor list +def test_cli_fre_cmor_list(): + ''' fre cmor list ''' + result = runner.invoke(fre.fre, args=["cmor", "list"]) + assert result.exit_code == 2 + +def test_cli_fre_cmor_list_help(): + ''' fre cmor list --help ''' + result = runner.invoke(fre.fre, args=["cmor", "list", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_cmor_list_opt_dne(): + ''' fre cmor list optionDNE ''' + result = runner.invoke(fre.fre, args=["cmor", "list", "optionDNE"]) + assert result.exit_code == 2 From b9e43b4b7dd3a896ce1a040d64186bb7aba36287 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 3 Dec 2024 17:49:40 -0500 Subject: [PATCH 20/36] actually add cmor_lister.py --- fre/cmor/cmor_lister.py | 109 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 fre/cmor/cmor_lister.py diff --git a/fre/cmor/cmor_lister.py b/fre/cmor/cmor_lister.py new file mode 100644 index 00000000..3c4d8557 --- /dev/null +++ b/fre/cmor/cmor_lister.py @@ -0,0 +1,109 @@ +''' fre cmor list +because ian got tired of typing things like the following in bash... + +varname=sos; \ +table_files=$(ls fre/tests/test_files/cmip6-cmor-tables/Tables/CMIP6_*.json); \ +for table_file in $table_files; do \ + echo $table_file; \ + cat $table_file | grep -A 10 "\"$varname\""; \ +done; + +''' + +#import os +import glob +import json +#import shutil +#import subprocess +from pathlib import Path + +import click + +def print_var_content( table_config_file = None, opt_var_name = None): + ''' one variable printing routine ''' + try: + proj_table_vars=json.load(table_config_file) + except Exception as exc: + raise Exception(f'problem getting proj_table_vars... WHY') + + var_content = None + try: + var_content = proj_table_vars["variable_entry"].get(opt_var_name) + except: + #print(f'(cmor_list_subtool) WARNING no "variable_entry" key. for {json_table_config}.' + # ' not the right json file probably. moving on!') + return + + if var_content is None: + #print(f'(cmor_list_subtool) variable {opt_var_name} not found in {Path(json_table_config).name}, moving on!') + return + + table_name = None + try: + print(f'(print_var_content) trying to get table_name from proj_table_vars...') + table_name = proj_table_vars["Header"].get(table_name) + except: + print(f'print_var_content) WARNING couldnt get header and table_name field') + pass + + print(f'(print_var_content) found {opt_var_name} data in table!\n') + if table_name is not None: + print(f' table_name = {table_name}') + + print(f' {opt_var_name} stuff is = \n') + for content in var_content: + print(f' {content}: {var_content[content]}') + print('\n') + + return + +def cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_var_name = None): + ''' + finds tables in the CMIP json config directory containing variable data of interest. prints it + out to screen, intended largely as a helper tool for cli users. + ''' + if not Path(json_table_config_dir).exists(): + raise OSError(f'(cmor_list_subtool) ERROR directory {json_table_config_dir} does not exist! exit.') + + print(f'(cmor_list_subtool) attempting to find and open files in dir: \n {json_table_config_dir} ') + json_table_configs=glob.glob(f'{json_table_config_dir}/CMIP6_*.json') + if json_table_configs is None: + raise OSError(f'ERROR directory {json_table_config_dir} contains no JSON files, exit.') + else: + print(f'(cmor_list_subtool) found content in json_table_config_dir')#: {json_table_configs}') + + var_list = None + if json_var_list is not None: + with open( json_var_list, "r", encoding = "utf-8") as var_list_file : + var_list=json.load(var_list_file) + + if opt_var_name is None and var_list is None: + raise ValueError(f'(cmor_list_subtool) ERROR: no opt_var_name given but also no content in variable list!!! exit!') + + if opt_var_name is not None: + for json_table_config in json_table_configs: + print(f'(cmor_list_subtool) attempting to open {json_table_config}') + with open( json_table_config, "r", encoding = "utf-8") as table_config_file: + print_var_content(table_config_file, opt_var_name) + + elif var_list is not None: + for var in var_list: + for json_table_config in json_table_configs: + #print(f'(cmor_list_subtool) attempting to open {json_table_config}') + with open( json_table_config, "r", encoding = "utf-8") as table_config_file: + #print(f' var = {var}, var_list[{var}]={var_list[var]}') + print_var_content(table_config_file, str(var_list[var])) + else: + print(f'(FATAL) this line should be unreachable!!!') + + return + + +@click.command() +def _cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_var_name = None): + ''' entry point to fre cmor run for click. see cmor_list_subtool for argument descriptions.''' + return cmor_list_subtool(json_var_list, json_table_config_dir, opt_var_name) + + +if __name__ == '__main__': + cmor_list_subtool() From 237402e3588fc0407e70a9e031bf0529eec6f9c9 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 6 Dec 2024 14:15:07 -0500 Subject: [PATCH 21/36] cmor lister updates, largely clean up of stdout text --- fre/cmor/cmor_lister.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/fre/cmor/cmor_lister.py b/fre/cmor/cmor_lister.py index 3c4d8557..95ebb8ec 100644 --- a/fre/cmor/cmor_lister.py +++ b/fre/cmor/cmor_lister.py @@ -19,8 +19,12 @@ import click -def print_var_content( table_config_file = None, opt_var_name = None): - ''' one variable printing routine ''' +DO_NOT_PRINT_LIST=[ 'comment', + 'ok_min_mean_abs', 'ok_max_mean_abs', + 'valid_min', 'valid_max' ] + +def print_var_content( table_config_file = None, var_name = None): + ''' one variable printing routine- looks for info regarding var_name in table_config_file ''' try: proj_table_vars=json.load(table_config_file) except Exception as exc: @@ -28,30 +32,35 @@ def print_var_content( table_config_file = None, opt_var_name = None): var_content = None try: - var_content = proj_table_vars["variable_entry"].get(opt_var_name) + var_content = proj_table_vars["variable_entry"].get(var_name) except: #print(f'(cmor_list_subtool) WARNING no "variable_entry" key. for {json_table_config}.' # ' not the right json file probably. moving on!') return if var_content is None: - #print(f'(cmor_list_subtool) variable {opt_var_name} not found in {Path(json_table_config).name}, moving on!') + #print(f'(cmor_list_subtool) variable {var_name} not found in {Path(json_table_config).name}, moving on!') return table_name = None try: - print(f'(print_var_content) trying to get table_name from proj_table_vars...') - table_name = proj_table_vars["Header"].get(table_name) + #print(f'(print_var_content) trying to get table_name from proj_table_vars...') + #print(f' table header is {proj_table_vars["Header"]}') + table_name = proj_table_vars["Header"].get('table_id').split(' ')[1] + #print(f' table_name = {table_name}') except: print(f'print_var_content) WARNING couldnt get header and table_name field') pass - print(f'(print_var_content) found {opt_var_name} data in table!\n') if table_name is not None: - print(f' table_name = {table_name}') - - print(f' {opt_var_name} stuff is = \n') - for content in var_content: + print(f'(print_var_content) found {var_name} data in table {table_name}!') + else: + print(f'(print_var_content) found {var_name} data in table, but not its table_name!') + + print(f' variable key: {var_name}') + for content in var_content: + if content in DO_NOT_PRINT_LIST: + continue print(f' {content}: {var_content[content]}') print('\n') @@ -80,13 +89,15 @@ def cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_v if opt_var_name is None and var_list is None: raise ValueError(f'(cmor_list_subtool) ERROR: no opt_var_name given but also no content in variable list!!! exit!') - if opt_var_name is not None: + if opt_var_name is not None: + print(f'(cmor_list_subtool) opt_var_name is not None: looking for only ONE variables worth of info!') for json_table_config in json_table_configs: - print(f'(cmor_list_subtool) attempting to open {json_table_config}') + #print(f'(cmor_list_subtool) attempting to open {json_table_config}') with open( json_table_config, "r", encoding = "utf-8") as table_config_file: print_var_content(table_config_file, opt_var_name) elif var_list is not None: + print(f'(cmor_list_subtool) opt_var_name is None, and var_list is not None, looking for many variables worth of info!') for var in var_list: for json_table_config in json_table_configs: #print(f'(cmor_list_subtool) attempting to open {json_table_config}') From ec61ef03697140ff643c749a30f0f5af724cb7ec Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 6 Dec 2024 14:15:52 -0500 Subject: [PATCH 22/36] break up reading bounds per each coordinate variable --- fre/cmor/cmor_mixer.py | 53 +++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 4f511a81..abc955d2 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -21,7 +21,6 @@ #### ------ helper functions ------ ### - def get_var_filenames(indir, var_filenames = None, local_var = None): ''' appends files ending in .nc located within indir to list var_filenames accepts three arguments @@ -202,28 +201,50 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, ## figure out the coordinate/dimension names programmatically TODO # Attempt to read lat coordinates - print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lat, lat_bnds') - lat, lat_bnds = None, None + print(f'(rewrite_netcdf_file_var) attempting to read coordinate, lat') + lat = None try: - lat, lat_bnds = ds["lat"][:], ds["lat_bnds"][:] + lat = ds["lat"][:] except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING could not read latitude coordinate. moving on.\n exc = {exc}') - print(f' lat = {lat}') - print(f' lat_bnds = {lat_bnds}') + print(f'(rewrite_netcdf_file_var) WARNING could not read latitude coordinate. moving on.\n exc = {exc}' + f' lat = {lat}' ) pass - print(f' DONE attempting to read coordinate(s), lat, lat_bnds') - + print(f' DONE attempting to read coordinate, lat') + + # Attempt to read lat coordinate bnds + print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lat_bnds') + lat_bnds = None + try: + lat_bnds = ds["lat_bnds"][:] + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not read latitude bnds. moving on.\n exc = {exc}' + f' lat_bnds = {lat_bnds}' ) + pass + print(f' DONE attempting to read lat coord bnds') + # Attempt to read lon coordinates - print(f'(rewrite_netcdf_file_var) attempting to read coordinate(s), lon, lon_bnds') - lon, lon_bnds = None, None + print(f'(rewrite_netcdf_file_var) attempting to read coordinate, lon') + lon = None + try: + lon = ds["lon"][:] + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING could not read longitude coordinate. moving on.\n exc = {exc}' + f' lon = {lon}' ) + pass + print(f' DONE attempting to read coordinate, lon') + + # Attempt to read lon coordinate bnds + print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lon_bnds') + lon_bnds = None try: - lon, lon_bnds = ds["lon"][:], ds["lon_bnds"][:] + lon_bnds = ds["lon_bnds"][:] except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING could not read longitude coordinate. moving on.\n exc = {exc}') - print(f' lon = {lon}') - print(f' lon_bnds = {lon_bnds}') + print(f'(rewrite_netcdf_file_var) WARNING could not read longitude bnds. moving on.\n exc = {exc}' + f' lon_bnds = {lon_bnds}' ) pass - print(f' DONE attempting to read coordinate(s), lon, lon_bnds') + print(f' DONE attempting to read lon coord bnds') + + #assert False # read in time_coords + units print(f'(rewrite_netcdf_file_var) attempting to read time_coords, and units...') From 87eccbfa6fec0c90954f41b6641c1800f0b6caba Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 6 Dec 2024 14:16:56 -0500 Subject: [PATCH 23/36] add notes regarding potential solution to lack of spherical lon/lat in native(tripolar)-grid ocean file case. the data one seeks is likely within the statics --- run_test_file_cases.py | 77 +++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/run_test_file_cases.py b/run_test_file_cases.py index e8fdfb17..0746a5f5 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -70,30 +70,43 @@ def run_cmor_RUN(filename, table, opt_var_name): return FOO_return -### THIS CASE MAY WORK if i rewrite the land file correctly -# 9) FAIL (4 dimensional data with no vertical) -# Result - type error, expecting landuse dimension to be string, not an int flag with interpretation -# Fix - rewrite the file and it's data, mapping it's landuse int flag coord to the desired string values -testfile_LUmip_refined_gr1_Emon_landusedim = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/LUmip_refined/ts/monthly/5yr/' + \ - 'LUmip_refined.185001-185412.gppLut.nc' + + +# 7) FAIL +# ocean, Omon / so +# Result - error, there's no coordinate called "olevel". only "z_i" and "z_l" available. +# from cmip6-cmor-tables CMIP6_coordinate.json, three candidates pointed to by olevel... +# 'depth_coord', 'ocean_sigma', 'ocean_sigma_z' +# from cmip6-cmor-tables CMIP6_formula_terms.json, two candidates pointed to by olevel... +# 'zlev', 'zlev_bnds' +# see https://github.com/search?q=repo%3APCMDI%2Fcmip6-cmor-tables%20olevel&type=code +testfile_ocean_monthly_z_1x1deg_gr = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_z_1x1deg.000101-000512.so.nc' try: - some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') + some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') except Exception as exc: print(f'exception caught: exc=\n{exc}') some_return=-1 pass -print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') +print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') if some_return != 0: - print('didnt pass the land-file test. exit.') - #sys.exit() + print('didnt pass ocean-file test number 2... exit.') +# sys.exit() -# 6) FAIL + +#### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. +#### for this case, that file is: +#### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +#### 'pp/ocean_monthly/' + \ +#### 'ocean_monthly.static.nc' +#### and that data is stored under "geolon" and "geolat" consuming dims "x" and "y". +# 6) FAIL # ocean, Omon / sos -# Result - error, +# Result - error, it wants lat/lon, but only xh, yh coordinates are available testfile_ocean_monthly_gn = \ '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ 'pp/ocean_monthly/ts/monthly/5yr/' + \ @@ -111,23 +124,25 @@ def run_cmor_RUN(filename, table, opt_var_name): -# 7) FAIL -# ocean, Omon / so -# Result - error, -testfile_ocean_monthly_z_1x1deg_gr = \ - '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_z_1x1deg.000101-000512.so.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') -if some_return != 0: - print('didnt pass ocean-file test number 2... exit.') -# sys.exit() + +#### THIS CASE MAY WORK if i rewrite the land file correctly, with the right landuse dimension strings +## 9) FAIL (4 dimensional data with no vertical) +## Result - type error, expecting landuse dimension to be string, not an int flag with interpretation +## Fix - rewrite the file and it's data, mapping it's landuse int flag coord to the desired string values +#testfile_LUmip_refined_gr1_Emon_landusedim = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' +#try: +# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') +#if some_return != 0: +# print('didnt pass the land-file test. exit.') +# #sys.exit() From e4be3d72cb9ae36e013588fcda31eac1ab534d0a Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 6 Dec 2024 14:18:32 -0500 Subject: [PATCH 24/36] rename scratchwork.py because i will likely need another scratchwork file to rewrite the ocean file when i get a chance... first, going to turn to olevel issues --- scratchwork.py => scratchwork_rewrite_gppLut_LUMIP_file.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scratchwork.py => scratchwork_rewrite_gppLut_LUMIP_file.py (100%) diff --git a/scratchwork.py b/scratchwork_rewrite_gppLut_LUMIP_file.py similarity index 100% rename from scratchwork.py rename to scratchwork_rewrite_gppLut_LUMIP_file.py From 4174f7203d2fab2cf2a21e19ec0717577bb9e603 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 6 Dec 2024 16:49:50 -0500 Subject: [PATCH 25/36] new function, from_dis_gimme_dis --- fre/cmor/cmor_mixer.py | 87 ++++++--------- run_test_file_cases.py | 247 +++++++++++++++++++++-------------------- 2 files changed, 162 insertions(+), 172 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index abc955d2..24e456c0 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -21,6 +21,23 @@ #### ------ helper functions ------ ### +def from_dis_gimme_dis(from_dis, gimme_dis): + ''' + gives you gimme_dis from from_dis. accepts two arguments, both mandatory. + from_dis: the target netCDF4.Dataset object to try reading from + gimme_dis: what from_dis is hopefully gonna have and you're gonna get + ''' + try: + return from_dis[gimme_dis][:].copy() + except Exception as exc: + print(f'(gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}' + f' from this: {from_dis} ' + f' exc = {exc}' + f' returning None!' ) + return None + + + def get_var_filenames(indir, var_filenames = None, local_var = None): ''' appends files ending in .nc located within indir to list var_filenames accepts three arguments @@ -202,67 +219,37 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # Attempt to read lat coordinates print(f'(rewrite_netcdf_file_var) attempting to read coordinate, lat') - lat = None - try: - lat = ds["lat"][:] - except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING could not read latitude coordinate. moving on.\n exc = {exc}' - f' lat = {lat}' ) - pass - print(f' DONE attempting to read coordinate, lat') - - # Attempt to read lat coordinate bnds + lat = from_dis_gimme_dis( from_dis = ds, + gimme_dis = "lat") print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lat_bnds') - lat_bnds = None - try: - lat_bnds = ds["lat_bnds"][:] - except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING could not read latitude bnds. moving on.\n exc = {exc}' - f' lat_bnds = {lat_bnds}' ) - pass - print(f' DONE attempting to read lat coord bnds') - - # Attempt to read lon coordinates + lat_bnds = from_dis_gimme_dis( from_dis = ds, + gimme_dis = "lat_bnds") print(f'(rewrite_netcdf_file_var) attempting to read coordinate, lon') - lon = None - try: - lon = ds["lon"][:] - except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING could not read longitude coordinate. moving on.\n exc = {exc}' - f' lon = {lon}' ) - pass - print(f' DONE attempting to read coordinate, lon') - - # Attempt to read lon coordinate bnds + lon = from_dis_gimme_dis( from_dis = ds, + gimme_dis = "lon") print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lon_bnds') - lon_bnds = None - try: - lon_bnds = ds["lon_bnds"][:] - except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING could not read longitude bnds. moving on.\n exc = {exc}' - f' lon_bnds = {lon_bnds}' ) - pass - print(f' DONE attempting to read lon coord bnds') - - #assert False + lon_bnds = from_dis_gimme_dis( from_dis = ds, + gimme_dis = "lon_bnds") # read in time_coords + units - print(f'(rewrite_netcdf_file_var) attempting to read time_coords, and units...') - time_coords = ds["time"][:] # out this in a try/except thingy, initializing like others? + print(f'(rewrite_netcdf_file_var) attempting to read coordinate time, and units...') + time_coords = from_dis_gimme_dis( from_dis = ds, + gimme_dis = 'time' ) + time_coord_units = ds["time"].units print(f" time_coord_units = {time_coord_units}") # read in time_bnds , if present - time_bnds = [] # shouldnt this be initialized like the others? - try: - time_bnds = ds["time_bnds"][:] - #print(f"(rewrite_netcdf_file_var) time_bnds = {time_bnds}") - except ValueError: - print( "(rewrite_netcdf_file_var) WARNING grabbing time_bnds didnt work... moving on") - + print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, time_bnds') + time_bnds = from_dis_gimme_dis( from_dis = ds, + gimme_dis = 'time_bnds' ) # read the input variable data, i believe - var = ds[target_var][:] + print(f'(rewrite_netcdf_file_var) attempting to read variable data, {target_var}') + var = from_dis_gimme_dis( from_dis = ds, + gimme_dis = target_var ) + #var = ds[target_var][:] + # grab var_dim var_dim = len(var.shape) diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 0746a5f5..2e273cb0 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -72,6 +72,113 @@ def run_cmor_RUN(filename, table, opt_var_name): + + + + +# 1) SUCCEEDs +# land, Lmon, gr1 +testfile_land_gr1_Lmon = \ + '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/land/ts/monthly/5yr/' + \ + 'land.005101-005512.lai.nc' +try: + some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') +except: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'land_gr1_Lmon / lai') + + +# 2) SUCCEEDs +# atmos, Amon / cl +testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.196001-196412.cl.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') + + +# 3) SUCCEEDs +# atmos, Amon / mc +testfile_atmos_level_cmip_gr1_Amon_fullL = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.195501-195912.mc.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') + + + +# 4) SUCCEEDs (no longitude coordinate case) +# atmos, AERmonZ / ta +# just like #1, but lack longitude +testfile_atmos_gr1_AERmonZ_nolons = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ + 'atmos_plev39_cmip.201001-201412.ta.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') + + +# 5) SUCCEEDs +# ocean, Omon / sos +testfile_ocean_monthly_1x1deg_gr = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_1x1deg.190001-190412.sos.nc' +try: + some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') + + + + +# 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) +# atmos, Amon / ch4global +testfile_atmos_scalar_gn_Amon_nolon_nolat = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_scalar/ts/monthly/5yr/' + \ + 'atmos_scalar.197001-197412.ch4global.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') + + + + + + + +sys.exit() + + + # 7) FAIL # ocean, Omon / so # Result - error, there's no coordinate called "olevel". only "z_i" and "z_l" available. @@ -125,125 +232,21 @@ def run_cmor_RUN(filename, table, opt_var_name): -#### THIS CASE MAY WORK if i rewrite the land file correctly, with the right landuse dimension strings -## 9) FAIL (4 dimensional data with no vertical) -## Result - type error, expecting landuse dimension to be string, not an int flag with interpretation -## Fix - rewrite the file and it's data, mapping it's landuse int flag coord to the desired string values -#testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/LUmip_refined/ts/monthly/5yr/' + \ -# 'LUmip_refined.185001-185412.gppLut.nc' -#try: -# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') -#if some_return != 0: -# print('didnt pass the land-file test. exit.') -# #sys.exit() - - - - -## 1) SUCCEEDs -## land, Lmon, gr1 -#testfile_land_gr1_Lmon = \ -# '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/land/ts/monthly/5yr/' + \ -# 'land.005101-005512.lai.nc' -#try: -# some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -#except: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'land_gr1_Lmon / lai') -# -# -## 2) SUCCEEDs -## atmos, Amon / cl -#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ -# 'atmos_level_cmip.196001-196412.cl.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') -# -# -## 3) SUCCEEDs -## atmos, Amon / mc -#testfile_atmos_level_cmip_gr1_Amon_fullL = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ -# 'atmos_level_cmip.195501-195912.mc.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') -# -# -# -## 4) SUCCEEDs (no longitude coordinate case) -## atmos, AERmonZ / ta -## just like #1, but lack longitude -#testfile_atmos_gr1_AERmonZ_nolons = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ -# 'atmos_plev39_cmip.201001-201412.ta.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') -# -# -## 5) SUCCEEDs -## ocean, Omon / sos -#testfile_ocean_monthly_1x1deg_gr = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ -# 'ocean_monthly_1x1deg.190001-190412.sos.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') -# -# -# -# -## 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) -## atmos, Amon / ch4global -#testfile_atmos_scalar_gn_Amon_nolon_nolat = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_scalar/ts/monthly/5yr/' + \ -# 'atmos_scalar.197001-197412.ch4global.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - - - - - - - - +### THIS CASE MAY WORK if i rewrite the land file correctly, with the right landuse dimension strings +# 9) FAIL (4 dimensional data with no vertical) +# Result - type error, expecting landuse dimension to be string, not an int flag with interpretation +# Fix - rewrite the file and it's data, mapping it's landuse int flag coord to the desired string values +testfile_LUmip_refined_gr1_Emon_landusedim = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/LUmip_refined/ts/monthly/5yr/' + \ + 'LUmip_refined.185001-185412.gppLut.nc' +try: + some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') +if some_return != 0: + print('didnt pass the land-file test. exit.') + #sys.exit() From 88055668bbe4c81e1b6cdcee39faf0f579e913f9 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 9 Dec 2024 15:22:52 -0500 Subject: [PATCH 26/36] new prototype scripts for rewriting the ocean file with CMOR compliant fields. rename prototype script for rewriting land file. miniscule edits to mixer itself --- fre/cmor/cmor_mixer.py | 9 +- run_test_file_cases.py | 272 +++++++++--------- ...k_rewrite_gppLut_LUmip_refined_gr1_file.py | 4 +- ...write_so_ocean_monthly_z_1x1deg_gr_file.py | 154 ++++++++++ ...hwork_rewrite_sos_ocean_monthly_gn_file.py | 154 ++++++++++ 5 files changed, 452 insertions(+), 141 deletions(-) rename scratchwork_rewrite_gppLut_LUMIP_file.py => scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py (98%) create mode 100644 scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py create mode 100644 scratchwork_rewrite_sos_ocean_monthly_gn_file.py diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 24e456c0..8791d960 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -30,8 +30,8 @@ def from_dis_gimme_dis(from_dis, gimme_dis): try: return from_dis[gimme_dis][:].copy() except Exception as exc: - print(f'(gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}' - f' from this: {from_dis} ' + print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}' + f' from this: {from_dis} ' f' exc = {exc}' f' returning None!' ) return None @@ -89,9 +89,10 @@ def check_dataset_for_ocean_grid(ds): ds: netCDF4.Dataset object containing variables with associated dimensional information. ''' if "xh" in list(ds.variables.keys()): + print("(check_dataset_for_ocean_grid) WARNING: 'xh' found in var_list: ocean grid req'd" + " sometimes i don't cmorize right! check me!") pass - #raise NotImplementedError( - print( "(check_dataset_for_ocean_grid) 'xh' found in var_list. ocean grid req'd but not yet unimplemented. stop.") + def get_vertical_dimension(ds, target_var): diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 2e273cb0..aeaf5c07 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -76,107 +76,107 @@ def run_cmor_RUN(filename, table, opt_var_name): -# 1) SUCCEEDs -# land, Lmon, gr1 -testfile_land_gr1_Lmon = \ - '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/land/ts/monthly/5yr/' + \ - 'land.005101-005512.lai.nc' -try: - some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -except: - print(f'exception caught: exc=\n{exc}') - some_return=-1 +## 1) SUCCEEDs +## land, Lmon, gr1 +#testfile_land_gr1_Lmon = \ +# '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/land/ts/monthly/5yr/' + \ +# 'land.005101-005512.lai.nc' +#try: +# some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') +#except: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'land_gr1_Lmon / lai') +# +# +## 2) SUCCEEDs +## atmos, Amon / cl +#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ +# 'atmos_level_cmip.196001-196412.cl.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') +# +# +## 3) SUCCEEDs +## atmos, Amon / mc +#testfile_atmos_level_cmip_gr1_Amon_fullL = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ +# 'atmos_level_cmip.195501-195912.mc.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') +# +# +# +## 4) SUCCEEDs (no longitude coordinate case) +## atmos, AERmonZ / ta +## just like #1, but lack longitude +#testfile_atmos_gr1_AERmonZ_nolons = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ +# 'atmos_plev39_cmip.201001-201412.ta.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') +# +# +## 5) SUCCEEDs +## ocean, Omon / sos +#testfile_ocean_monthly_1x1deg_gr = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ +# 'ocean_monthly_1x1deg.190001-190412.sos.nc' +#try: +# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 # pass -print_the_outcome(some_return,'land_gr1_Lmon / lai') - - -# 2) SUCCEEDs -# atmos, Amon / cl -testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.196001-196412.cl.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') - - -# 3) SUCCEEDs -# atmos, Amon / mc -testfile_atmos_level_cmip_gr1_Amon_fullL = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.195501-195912.mc.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') - - - -# 4) SUCCEEDs (no longitude coordinate case) -# atmos, AERmonZ / ta -# just like #1, but lack longitude -testfile_atmos_gr1_AERmonZ_nolons = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ - 'atmos_plev39_cmip.201001-201412.ta.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') - - -# 5) SUCCEEDs -# ocean, Omon / sos -testfile_ocean_monthly_1x1deg_gr = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_1x1deg.190001-190412.sos.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') - +#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') +# +# +# +# +## 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) +## atmos, Amon / ch4global +#testfile_atmos_scalar_gn_Amon_nolon_nolat = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_scalar/ts/monthly/5yr/' + \ +# 'atmos_scalar.197001-197412.ch4global.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') -# 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) -# atmos, Amon / ch4global -testfile_atmos_scalar_gn_Amon_nolon_nolat = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_scalar/ts/monthly/5yr/' + \ - 'atmos_scalar.197001-197412.ch4global.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - -sys.exit() - # 7) FAIL @@ -205,48 +205,48 @@ def run_cmor_RUN(filename, table, opt_var_name): -#### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. -#### for this case, that file is: -#### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -#### 'pp/ocean_monthly/' + \ -#### 'ocean_monthly.static.nc' -#### and that data is stored under "geolon" and "geolat" consuming dims "x" and "y". -# 6) FAIL -# ocean, Omon / sos -# Result - error, it wants lat/lon, but only xh, yh coordinates are available -testfile_ocean_monthly_gn = \ - '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/ocean_monthly/ts/monthly/5yr/' + \ - 'ocean_monthly.002101-002512.sos.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'ocean_monthly_gn / sos') -if some_return != 0: - print('didnt pass ocean-file test number 1... exit.') -# sys.exit() - - - - -### THIS CASE MAY WORK if i rewrite the land file correctly, with the right landuse dimension strings -# 9) FAIL (4 dimensional data with no vertical) -# Result - type error, expecting landuse dimension to be string, not an int flag with interpretation -# Fix - rewrite the file and it's data, mapping it's landuse int flag coord to the desired string values -testfile_LUmip_refined_gr1_Emon_landusedim = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/LUmip_refined/ts/monthly/5yr/' + \ - 'LUmip_refined.185001-185412.gppLut.nc' -try: - some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') -if some_return != 0: - print('didnt pass the land-file test. exit.') - #sys.exit() +##### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. +##### for this case, that file is: +##### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +##### 'pp/ocean_monthly/' + \ +##### 'ocean_monthly.static.nc' +##### and that data is stored under "geolon" and "geolat" consuming dims "x" and "y". +## 6) FAIL +## ocean, Omon / sos +## Result - error, it wants lat/lon, but only xh, yh coordinates are available +#testfile_ocean_monthly_gn = \ +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly/ts/monthly/5yr/' + \ +# 'ocean_monthly.002101-002512.sos.nc' +#try: +# some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'ocean_monthly_gn / sos') +#if some_return != 0: +# print('didnt pass ocean-file test number 1... exit.') +## sys.exit() +# +# +# +# +#### THIS CASE MAY WORK if i rewrite the land file correctly, with the right landuse dimension strings +## 9) FAIL (4 dimensional data with no vertical) +## Result - type error, expecting landuse dimension to be string, not an int flag with interpretation +## Fix - rewrite the file and it's data, mapping it's landuse int flag coord to the desired string values +#testfile_LUmip_refined_gr1_Emon_landusedim = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' +#try: +# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') +#if some_return != 0: +# print('didnt pass the land-file test. exit.') +# #sys.exit() diff --git a/scratchwork_rewrite_gppLut_LUMIP_file.py b/scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py similarity index 98% rename from scratchwork_rewrite_gppLut_LUMIP_file.py rename to scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py index 8abfaa8d..edb78242 100644 --- a/scratchwork_rewrite_gppLut_LUMIP_file.py +++ b/scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py @@ -18,7 +18,9 @@ landuse_str_list=[b"primary_and_secondary_land", b"pastures", b"crops", b"urban"] -# open netcdf file in append mode? write mode? read+? +# open netcdf file in append mode? write mode? read? +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ gppLut_fin=Dataset('./tmp/LUmip_refined.185001-185412.gppLut.nc', mode='r') gppLut_fin_ncattrs=gppLut_fin.__dict__ #dictionary diff --git a/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py b/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py new file mode 100644 index 00000000..48e06555 --- /dev/null +++ b/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +#import sys +#import os +from pathlib import Path + +import numpy +#from numpy.dtypes import StringDType +from netCDF4 import Dataset #, stringtochar + + + +# open netcdf file in append mode? write mode? read? +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ +so_gr_fin=Dataset('./tmp/ocean_monthly_z_1x1deg.000101-000512.so.nc', mode='r') +so_gr_fin_ncattrs=so_gr_fin.__dict__ #dictionary + +# the target data of interest +so_gr_var_data = so_gr_fin.variables['so'][:] +so_gr_var_atts = so_gr_fin.variables['so'].__dict__ + +# coordinate variables, their _bnds, and their identically named dimensions +# coordinate variable == a variable with the same name as a dimension. +# pitfall: an "axis" in netcdf is not analagous to a dimension, overloaded term +bnds_coord_data = so_gr_fin.variables['bnds'][:] +bnds_coord_atts = so_gr_fin.variables['bnds'].__dict__ +bnds_coord_dims = so_gr_fin.dimensions['bnds'].size + +time_coord_data = so_gr_fin.variables['time'][:] +time_coord_atts = so_gr_fin.variables['time'].__dict__ +time_coord_bnds = so_gr_fin.variables['time_bnds'][:] +time_coord_bnds_atts = so_gr_fin.variables['time_bnds'].__dict__ +#time_coord_dims = so_gr_fin.dimensions['time'].size + +lat_coord_data = so_gr_fin.variables['lat'][:] +lat_coord_atts = so_gr_fin.variables['lat'].__dict__ +lat_coord_bnds = so_gr_fin.variables['lat_bnds'][:] +lat_coord_bnds_atts = so_gr_fin.variables['lat_bnds'].__dict__ +lat_coord_dims = so_gr_fin.dimensions['lat'].size + +lon_coord_data = so_gr_fin.variables['lon'][:] +lon_coord_atts = so_gr_fin.variables['lon'].__dict__ +lon_coord_bnds = so_gr_fin.variables['lon_bnds'][:] +lon_coord_bnds_atts = so_gr_fin.variables['lon_bnds'].__dict__ +lon_coord_dims = so_gr_fin.dimensions['lon'].size + +''' + we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work + recall, a netCDF4 file is, basically, 4 sets of things + attributes, i.e effectively global metadata + groups, i.e. a heirarchy with nesting a lot like directories (older netcdf files only have a root group) + dimensions, i.e. a set of named-integers to define the number of divisions on an axis + variables, i.e. arrays representing data with shapes described by the dimensions in the file +''' + +# open the output file +so_gr_fout=Dataset('./alt_so_gr_input/PLAY_ocean_monthly_z_1x1deg.000101-000512.so.nc',mode='w') +so_gr_fout.setncatts(so_gr_fin_ncattrs) + +''' + from netCDF4 python API doc, for easy referencing + createDimension(self, + dimname, size=None)... None will imply unlimited +''' +so_gr_fout.createDimension( 'time', + None ) #time_coord_dims +so_gr_fout.createDimension( 'bnds', + bnds_coord_dims ) + +so_gr_fout.createDimension( 'lat', + lat_coord_dims ) +so_gr_fout.createDimension( 'lon', + lon_coord_dims ) + + + + +''' + from netCDF4 python API doc, for easy referencing + def createVariable(self, + varname, datatype, dimensions=(), + lots others ) +''' +# easy variables first. +# bnds +so_gr_fout.createVariable( 'bnds', so_gr_fin.variables['bnds'].dtype, + dimensions = ( so_gr_fout.dimensions['bnds'] ) ) +so_gr_fout.variables['bnds'][:] = bnds_coord_data +so_gr_fout.variables['bnds'].setncatts( bnds_coord_atts ) + +# time +so_gr_fout.createVariable( 'time', so_gr_fin.variables['time'].dtype, + dimensions = ( so_gr_fout.dimensions['time'] ) ) +so_gr_fout.variables['time'][:] = time_coord_data +so_gr_fout.variables['time'].setncatts( time_coord_atts ) + +# time_bnds +so_gr_fout.createVariable( 'time_bnds', so_gr_fin.variables['time_bnds'].dtype, + fill_value = so_gr_fin.variables['time_bnds']._FillValue, # necessary bc of unlimited + extra limited dim shape? + dimensions = ( so_gr_fout.dimensions['time'], + so_gr_fout.dimensions['bnds'] ) ) +so_gr_fout.variables['time_bnds'][:] = time_coord_bnds +for att in time_coord_bnds_atts: #so_gr_fout.variables['time_bnds'].setncatts( time_coord_bnds_atts ) + if att != '_FillValue': + so_gr_fout.variables['time_bnds'].setncattr( att, time_coord_bnds_atts[att] ) + +# lat +so_gr_fout.createVariable( 'lat', so_gr_fin.variables['lat'].dtype, + dimensions = ( so_gr_fout.dimensions['lat'] ) ) +so_gr_fout.variables['lat'][:] = lat_coord_data +so_gr_fout.variables['lat'].setncatts( lat_coord_atts ) + +# lat_bnds +so_gr_fout.createVariable( 'lat_bnds', so_gr_fin.variables['lat_bnds'].dtype, + dimensions = ( so_gr_fout.dimensions['lat'], + so_gr_fout.dimensions['bnds'] ) ) +so_gr_fout.variables['lat_bnds'][:] = lat_coord_bnds +so_gr_fout.variables['lat_bnds'].setncatts( lat_coord_bnds_atts ) + +# lon +so_gr_fout.createVariable( 'lon', so_gr_fin.variables['lon'].dtype, + dimensions = ( so_gr_fout.dimensions['lon'] ) ) +so_gr_fout.variables['lon'][:] = lon_coord_data +so_gr_fout.variables['lon'].setncatts( lon_coord_atts ) + +# lon_bnds +so_gr_fout.createVariable( 'lon_bnds', so_gr_fin.variables['lon_bnds'].dtype, + dimensions = ( so_gr_fout.dimensions['lon'], + so_gr_fout.dimensions['bnds'] ) ) +so_gr_fout.variables['lon_bnds'][:] = lon_coord_bnds +so_gr_fout.variables['lon_bnds'].setncatts( lon_coord_bnds_atts ) + +# data time!!! +so_gr_fout.createVariable( 'so', so_gr_fin.variables['so'].dtype, + fill_value = so_gr_fin.variables['so']._FillValue, + dimensions = ( so_gr_fout.dimensions['time'], + None, #TODO SHOULD NOT BE NONE!!!! + so_gr_fout.dimensions['lat'], + so_gr_fout.dimensions['lon'] ) ) +so_gr_fout.variables['so'][:] = so_gr_var_data +for att in so_gr_var_atts: + if att not in ["time_avg_info", "_FillValue"]: + so_gr_fout.variables['so'].setncattr(att, so_gr_var_atts[att] ) + +so_gr_fout.close() + +## test that the two are equivalent "quickly"... +#unmsk_so_gr_var_data=so_gr_var_data[~so_gr_var_data.mask] +#unmsk_so_gr_var_out=so_gr_var_out[~so_gr_var_out.mask] +#for i in range(0, len( unmsk_so_gr_var_data ) ): +# if i%100 == 0: +# print(f'i = {i}') +# diff = unmsk_so_gr_var_data[i] - unmsk_so_gr_var_out[i] +# if diff > 0.: +# print(f'diff = \n {diff}') diff --git a/scratchwork_rewrite_sos_ocean_monthly_gn_file.py b/scratchwork_rewrite_sos_ocean_monthly_gn_file.py new file mode 100644 index 00000000..04f7167f --- /dev/null +++ b/scratchwork_rewrite_sos_ocean_monthly_gn_file.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +#import sys +#import os +from pathlib import Path + +import numpy +#from numpy.dtypes import StringDType +from netCDF4 import Dataset #, stringtochar + + + +# open netcdf file in append mode? write mode? read? +# +# +sos_gn_fin=Dataset('./tmp/.nc', mode='r') +sos_gn_fin_ncattrs=sos_gn_fin.__dict__ #dictionary + +# the target data of interest +sos_gn_var_data = sos_gn_fin.variables['sos'][:] +sos_gn_var_atts = sos_gn_fin.variables['sos'].__dict__ + +# coordinate variables, their _bnds, and their identically named dimensions +# coordinate variable == a variable with the same name as a dimension. +# pitfall: an "axis" in netcdf is not analagous to a dimension, overloaded term +bnds_coord_data = sos_gn_fin.variables['bnds'][:] +bnds_coord_atts = sos_gn_fin.variables['bnds'].__dict__ +bnds_coord_dims = sos_gn_fin.dimensions['bnds'].size + +time_coord_data = sos_gn_fin.variables['time'][:] +time_coord_atts = sos_gn_fin.variables['time'].__dict__ +time_coord_bnds = sos_gn_fin.variables['time_bnds'][:] +time_coord_bnds_atts = sos_gn_fin.variables['time_bnds'].__dict__ +#time_coord_dims = sos_gn_fin.dimensions['time'].size + +lat_coord_data = sos_gn_fin.variables['lat'][:] +lat_coord_atts = sos_gn_fin.variables['lat'].__dict__ +lat_coord_bnds = sos_gn_fin.variables['lat_bnds'][:] +lat_coord_bnds_atts = sos_gn_fin.variables['lat_bnds'].__dict__ +lat_coord_dims = sos_gn_fin.dimensions['lat'].size + +lon_coord_data = sos_gn_fin.variables['lon'][:] +lon_coord_atts = sos_gn_fin.variables['lon'].__dict__ +lon_coord_bnds = sos_gn_fin.variables['lon_bnds'][:] +lon_coord_bnds_atts = sos_gn_fin.variables['lon_bnds'].__dict__ +lon_coord_dims = sos_gn_fin.dimensions['lon'].size + +''' + we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work + recall, a netCDF4 file is, basically, 4 sets of things + attributes, i.e effectively global metadata + groups, i.e. a heirarchy with nesting a lot like directories (older netcdf files only have a root group) + dimensions, i.e. a set of named-integers to define the number of divisions on an axis + variables, i.e. arrays representing data with shapes described by the dimensions in the file +''' + +# open the output file +sos_gn_fout=Dataset('./alt_sos_gn_input/PLAY_.nc',mode='w') +sos_gn_fout.setncatts(sos_gn_fin_ncattrs) + +''' + from netCDF4 python API doc, for easy referencing + createDimension(self, + dimname, size=None)... None will imply unlimited +''' +sos_gn_fout.createDimension( 'time', + None ) #time_coord_dims +sos_gn_fout.createDimension( 'bnds', + bnds_coord_dims ) + +sos_gn_fout.createDimension( 'lat', + lat_coord_dims ) +sos_gn_fout.createDimension( 'lon', + lon_coord_dims ) + + + + +''' + from netCDF4 python API doc, for easy referencing + def createVariable(self, + varname, datatype, dimensions=(), + lots others ) +''' +# easy variables first. +# bnds +sos_gn_fout.createVariable( 'bnds', sos_gn_fin.variables['bnds'].dtype, + dimensions = ( sos_gn_fout.dimensions['bnds'] ) ) +sos_gn_fout.variables['bnds'][:] = bnds_coord_data +sos_gn_fout.variables['bnds'].setncatts( bnds_coord_atts ) + +# time +sos_gn_fout.createVariable( 'time', sos_gn_fin.variables['time'].dtype, + dimensions = ( sos_gn_fout.dimensions['time'] ) ) +sos_gn_fout.variables['time'][:] = time_coord_data +sos_gn_fout.variables['time'].setncatts( time_coord_atts ) + +# time_bnds +sos_gn_fout.createVariable( 'time_bnds', sos_gn_fin.variables['time_bnds'].dtype, + fill_value = sos_gn_fin.variables['time_bnds']._FillValue, # necessary bc of unlimited + extra limited dim shape? + dimensions = ( sos_gn_fout.dimensions['time'], + sos_gn_fout.dimensions['bnds'] ) ) +sos_gn_fout.variables['time_bnds'][:] = time_coord_bnds +for att in time_coord_bnds_atts: #sos_gn_fout.variables['time_bnds'].setncatts( time_coord_bnds_atts ) + if att != '_FillValue': + sos_gn_fout.variables['time_bnds'].setncattr( att, time_coord_bnds_atts[att] ) + +# lat +sos_gn_fout.createVariable( 'lat', sos_gn_fin.variables['lat'].dtype, + dimensions = ( sos_gn_fout.dimensions['lat'] ) ) +sos_gn_fout.variables['lat'][:] = lat_coord_data +sos_gn_fout.variables['lat'].setncatts( lat_coord_atts ) + +# lat_bnds +sos_gn_fout.createVariable( 'lat_bnds', sos_gn_fin.variables['lat_bnds'].dtype, + dimensions = ( sos_gn_fout.dimensions['lat'], + sos_gn_fout.dimensions['bnds'] ) ) +sos_gn_fout.variables['lat_bnds'][:] = lat_coord_bnds +sos_gn_fout.variables['lat_bnds'].setncatts( lat_coord_bnds_atts ) + +# lon +sos_gn_fout.createVariable( 'lon', sos_gn_fin.variables['lon'].dtype, + dimensions = ( sos_gn_fout.dimensions['lon'] ) ) +sos_gn_fout.variables['lon'][:] = lon_coord_data +sos_gn_fout.variables['lon'].setncatts( lon_coord_atts ) + +# lon_bnds +sos_gn_fout.createVariable( 'lon_bnds', sos_gn_fin.variables['lon_bnds'].dtype, + dimensions = ( sos_gn_fout.dimensions['lon'], + sos_gn_fout.dimensions['bnds'] ) ) +sos_gn_fout.variables['lon_bnds'][:] = lon_coord_bnds +sos_gn_fout.variables['lon_bnds'].setncatts( lon_coord_bnds_atts ) + +# data time!!! +sos_gn_fout.createVariable( 'sos', sos_gn_fin.variables['sos'].dtype, + fill_value = sos_gn_fin.variables['sos']._FillValue, + dimensions = ( sos_gn_fout.dimensions['time'], + None, #TODO SHOULD NOT BE NONE!!!! + sos_gn_fout.dimensions['lat'], + sos_gn_fout.dimensions['lon'] ) ) +sos_gn_fout.variables['sos'][:] = sos_gn_var_data +for att in sos_gn_var_atts: + if att not in ["time_avg_info", "_FillValue"]: + sos_gn_fout.variables['sos'].setncattr(att, sos_gn_var_atts[att] ) + +sos_gn_fout.close() + +## test that the two are equivalent "quickly"... +#unmsk_sos_gn_var_data=sos_gn_var_data[~sos_gn_var_data.mask] +#unmsk_sos_gn_var_out=sos_gn_var_out[~sos_gn_var_out.mask] +#for i in range(0, len( unmsk_sos_gn_var_data ) ): +# if i%100 == 0: +# print(f'i = {i}') +# diff = unmsk_sos_gn_var_data[i] - unmsk_sos_gn_var_out[i] +# if diff > 0.: +# print(f'diff = \n {diff}') From 6e2dfa33b92a6e596fd44eb893fe31731562e4f5 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 10 Dec 2024 13:59:01 -0500 Subject: [PATCH 27/36] dinking around with rewriting ocean file --- ...write_so_ocean_monthly_z_1x1deg_gr_file.py | 58 +++++++++++++++---- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py b/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py index 48e06555..72ee48e0 100644 --- a/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py +++ b/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py @@ -8,6 +8,12 @@ from netCDF4 import Dataset #, stringtochar +## we can see from +# ncdump -v lev /data_cmip6/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/piControl/r1i1p1f1/Omon/so/gr/v20180701/so_Omon_GFDL-ESM4_piControl_r1i1p1f1_gr_000101-002012.nc | grep -A 10 'lev = ' +# ncdump -v lev /data_cmip6/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/piControl/r1i1p1f1/Omon/so/gn/v20180701/so_Omon_GFDL-ESM4_piControl_r1i1p1f1_gn_000101-002012.nc | grep -A 10 'lev = ' +# ncdump -v lev /data_cmip6/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/piControl/r1i1p1f1/Omon/so/gn/v20180701/so_Omon_GFDL-ESM4_piControl_r1i1p1f1_gn_000101-002012.nc | grep -A 10 'lev(' +# ncdump -v lev /data_cmip6/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/piControl/r1i1p1f1/Omon/so/gr/v20180701/so_Omon_GFDL-ESM4_piControl_r1i1p1f1_gr_000101-002012.nc | grep -A 10 'lev(' +# that 'z_l' simply needs to be renamed, effectively. # open netcdf file in append mode? write mode? read? # '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ @@ -22,8 +28,9 @@ # coordinate variables, their _bnds, and their identically named dimensions # coordinate variable == a variable with the same name as a dimension. # pitfall: an "axis" in netcdf is not analagous to a dimension, overloaded term -bnds_coord_data = so_gr_fin.variables['bnds'][:] -bnds_coord_atts = so_gr_fin.variables['bnds'].__dict__ + +# N/A #bnds_coord_data = so_gr_fin.variables['bnds'][:] +# N/A #bnds_coord_atts = so_gr_fin.variables['bnds'].__dict__ bnds_coord_dims = so_gr_fin.dimensions['bnds'].size time_coord_data = so_gr_fin.variables['time'][:] @@ -44,6 +51,15 @@ lon_coord_bnds_atts = so_gr_fin.variables['lon_bnds'].__dict__ lon_coord_dims = so_gr_fin.dimensions['lon'].size +z_l_coord_data = so_gr_fin.variables['z_l'][:] +z_l_coord_atts = so_gr_fin.variables['z_l'].__dict__ +# N/A #z_l_coord_bnds = so_gr_fin.variables['lon_bnds'][:] +# N/A #z_l_coord_bnds_atts = so_gr_fin.variables['lon_bnds'].__dict__ +z_l_coord_dims = so_gr_fin.dimensions['z_l'].size + + + + ''' we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work recall, a netCDF4 file is, basically, 4 sets of things @@ -54,7 +70,7 @@ ''' # open the output file -so_gr_fout=Dataset('./alt_so_gr_input/PLAY_ocean_monthly_z_1x1deg.000101-000512.so.nc',mode='w') +so_gr_fout=Dataset('./alt_Omon_so_gr_input/PLAY_ocean_monthly_z_1x1deg.000101-000512.so.nc',mode='w') so_gr_fout.setncatts(so_gr_fin_ncattrs) ''' @@ -72,6 +88,10 @@ so_gr_fout.createDimension( 'lon', lon_coord_dims ) +#so_gr_fout.createDimension( 'lev', +so_gr_fout.createDimension( 'olevel', + z_l_coord_dims ) + @@ -80,12 +100,12 @@ def createVariable(self, varname, datatype, dimensions=(), + lots others ) ''' -# easy variables first. -# bnds -so_gr_fout.createVariable( 'bnds', so_gr_fin.variables['bnds'].dtype, - dimensions = ( so_gr_fout.dimensions['bnds'] ) ) -so_gr_fout.variables['bnds'][:] = bnds_coord_data -so_gr_fout.variables['bnds'].setncatts( bnds_coord_atts ) +# N/A ## easy variables first. +# N/A ## bnds +# N/A #so_gr_fout.createVariable( 'bnds', so_gr_fin.variables['bnds'].dtype, +# N/A # dimensions = ( so_gr_fout.dimensions['bnds'] ) ) +# N/A #so_gr_fout.variables['bnds'][:] = bnds_coord_data +# N/A #so_gr_fout.variables['bnds'].setncatts( bnds_coord_atts ) # time so_gr_fout.createVariable( 'time', so_gr_fin.variables['time'].dtype, @@ -95,11 +115,13 @@ def createVariable(self, # time_bnds so_gr_fout.createVariable( 'time_bnds', so_gr_fin.variables['time_bnds'].dtype, - fill_value = so_gr_fin.variables['time_bnds']._FillValue, # necessary bc of unlimited + extra limited dim shape? + # N/A #fill_value = so_gr_fin.variables['time_bnds']._FillValue, # necessary bc of unlimited + extra limited dim shape? dimensions = ( so_gr_fout.dimensions['time'], so_gr_fout.dimensions['bnds'] ) ) so_gr_fout.variables['time_bnds'][:] = time_coord_bnds for att in time_coord_bnds_atts: #so_gr_fout.variables['time_bnds'].setncatts( time_coord_bnds_atts ) + print("HELLO???") + print(f'att = {att}') if att != '_FillValue': so_gr_fout.variables['time_bnds'].setncattr( att, time_coord_bnds_atts[att] ) @@ -129,11 +151,25 @@ def createVariable(self, so_gr_fout.variables['lon_bnds'][:] = lon_coord_bnds so_gr_fout.variables['lon_bnds'].setncatts( lon_coord_bnds_atts ) + +## lev +#so_gr_fout.createVariable( 'lev', so_gr_fin.variables['z_l'].dtype, +# dimensions = ( so_gr_fout.dimensions['lev'] ) ) +#so_gr_fout.variables['lev'][:] = z_l_coord_data +#so_gr_fout.variables['lev'].setncatts( z_l_coord_atts ) + +# olevel +so_gr_fout.createVariable( 'olevel', so_gr_fin.variables['z_l'].dtype, + dimensions = ( so_gr_fout.dimensions['olevel'] ) ) +so_gr_fout.variables['olevel'][:] = z_l_coord_data +so_gr_fout.variables['olevel'].setncatts( z_l_coord_atts ) + + # data time!!! so_gr_fout.createVariable( 'so', so_gr_fin.variables['so'].dtype, fill_value = so_gr_fin.variables['so']._FillValue, dimensions = ( so_gr_fout.dimensions['time'], - None, #TODO SHOULD NOT BE NONE!!!! + so_gr_fout.dimensions['olevel'], #so_gr_fout.dimensions['lev'], #TODO SHOULD NOT BE NONE!!!! so_gr_fout.dimensions['lat'], so_gr_fout.dimensions['lon'] ) ) so_gr_fout.variables['so'][:] = so_gr_var_data From 459efaeb838a6f9aef364d186875d292801a94b4 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 10 Dec 2024 13:59:41 -0500 Subject: [PATCH 28/36] forget rewriting the whole ocean file- pretty sure all this needs is a coordinate named depth_coord that has bounds. if i give cmor the values, it should do the rest --- fre/cmor/cmor_mixer.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 8791d960..00fad903 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -11,6 +11,8 @@ import subprocess from pathlib import Path +import numpy as np + import netCDF4 as nc import click import cmor @@ -36,7 +38,22 @@ def from_dis_gimme_dis(from_dis, gimme_dis): f' returning None!' ) return None +def create_lev_bnds(bound_these = None, with_these = None): + the_bnds = None + assert len(with_these) == len(bound_these) + 1 + print(f'(create_lev_bnds) bound_these is... ') + print(f' bound_these = \n{bound_these}') + print(f'(create_lev_bnds) with_these is... ') + print(f' with_these = \n{with_these}') + + the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these),2) + for i in range(0,len(bound_these)): + the_bnds[i][0]=with_these[i] + the_bnds[i][1]=with_these[i+1] + print(f'(create_lev_bnds) the_bnds is... ') + print(f' the_bnds = \n{the_bnds}') + return the_bnds def get_var_filenames(indir, var_filenames = None, local_var = None): ''' @@ -91,7 +108,8 @@ def check_dataset_for_ocean_grid(ds): if "xh" in list(ds.variables.keys()): print("(check_dataset_for_ocean_grid) WARNING: 'xh' found in var_list: ocean grid req'd" " sometimes i don't cmorize right! check me!") - pass + return True + return False @@ -202,7 +220,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # ocean grids are not implemented yet. print( '(rewrite_netcdf_file_var) checking input netcdf file for oceangrid condition') - check_dataset_for_ocean_grid(ds) + uses_ocean_grid = check_dataset_for_ocean_grid(ds) # try to read what coordinate(s) we're going to be expecting for the variable @@ -267,8 +285,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # Check var_dim and vert_dim and assign lev if relevant. # error if vert_dim wrong given var_dim lev, lev_units = None, "1" #1 #"none" #None #"" + lev_bnds = None if vert_dim != 0: - if vert_dim.lower() not in [ "landuse", "plev39", "plev30", "plev19", "plev8", + if vert_dim.lower() not in [ "z_l", "landuse", "plev39", "plev30", "plev19", "plev8", "height2m", "level", "lev", "levhalf"] : raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') lev = ds[vert_dim] @@ -368,6 +387,15 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') cmor_lev = cmor.axis( cmor_vert_dim_name, coord_vals = lev[:], units = lev_units ) + elif vert_dim in ["z_l"]: + lev_bnds = create_lev_bnds( bound_these = lev, + with_these = ds['z_i'] ) + print('(rewrite_netcdf_file_var) created lev_bnds...') + print(f' lev_bnds = \n{lev_bnds}') + cmor_lev = cmor.axis( 'depth_coord', + coord_vals = lev[:], + units = lev_units, + cell_bounds = lev_bnds) elif vert_dim in ["level", "lev", "levhalf"]: # find the ps file nearby From 920a3611a761630208670e4ecd05df815e6e3ad2 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 10 Dec 2024 14:00:23 -0500 Subject: [PATCH 29/36] ... dun dun dun... another one bites the dust... --- run_test_file_cases.py | 257 +++++++++++++++++++++++++---------------- 1 file changed, 156 insertions(+), 101 deletions(-) diff --git a/run_test_file_cases.py b/run_test_file_cases.py index aeaf5c07..065d1dfd 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -76,117 +76,85 @@ def run_cmor_RUN(filename, table, opt_var_name): -## 1) SUCCEEDs -## land, Lmon, gr1 -#testfile_land_gr1_Lmon = \ -# '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/land/ts/monthly/5yr/' + \ -# 'land.005101-005512.lai.nc' -#try: -# some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -#except: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'land_gr1_Lmon / lai') -# -# -## 2) SUCCEEDs -## atmos, Amon / cl -#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ -# 'atmos_level_cmip.196001-196412.cl.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') -# -# -## 3) SUCCEEDs -## atmos, Amon / mc -#testfile_atmos_level_cmip_gr1_Amon_fullL = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ -# 'atmos_level_cmip.195501-195912.mc.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') -# -# -# -## 4) SUCCEEDs (no longitude coordinate case) -## atmos, AERmonZ / ta -## just like #1, but lack longitude -#testfile_atmos_gr1_AERmonZ_nolons = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ -# 'atmos_plev39_cmip.201001-201412.ta.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') -# -# -## 5) SUCCEEDs -## ocean, Omon / sos -#testfile_ocean_monthly_1x1deg_gr = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ -# 'ocean_monthly_1x1deg.190001-190412.sos.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 +# 1) SUCCEEDs +# land, Lmon, gr1 +testfile_land_gr1_Lmon = \ + '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/land/ts/monthly/5yr/' + \ + 'land.005101-005512.lai.nc' +try: + some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') +except: + print(f'exception caught: exc=\n{exc}') + some_return=-1 # pass -#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') -# -# -# -# -## 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) -## atmos, Amon / ch4global -#testfile_atmos_scalar_gn_Amon_nolon_nolat = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_scalar/ts/monthly/5yr/' + \ -# 'atmos_scalar.197001-197412.ch4global.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') +print_the_outcome(some_return,'land_gr1_Lmon / lai') +# 2) SUCCEEDs +# atmos, Amon / cl +testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.196001-196412.cl.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') +# 3) SUCCEEDs +# atmos, Amon / mc +testfile_atmos_level_cmip_gr1_Amon_fullL = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.195501-195912.mc.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') +# 4) SUCCEEDs (no longitude coordinate case) +# atmos, AERmonZ / ta +# just like #1, but lack longitude +testfile_atmos_gr1_AERmonZ_nolons = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ + 'atmos_plev39_cmip.201001-201412.ta.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') +# 5) SUCCEEDs +# ocean, Omon / sos +testfile_ocean_monthly_1x1deg_gr = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_1x1deg.190001-190412.sos.nc' +try: + some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') -# 7) FAIL +# 7) SUCCEEDs # ocean, Omon / so -# Result - error, there's no coordinate called "olevel". only "z_i" and "z_l" available. -# from cmip6-cmor-tables CMIP6_coordinate.json, three candidates pointed to by olevel... -# 'depth_coord', 'ocean_sigma', 'ocean_sigma_z' -# from cmip6-cmor-tables CMIP6_formula_terms.json, two candidates pointed to by olevel... -# 'zlev', 'zlev_bnds' -# see https://github.com/search?q=repo%3APCMDI%2Fcmip6-cmor-tables%20olevel&type=code testfile_ocean_monthly_z_1x1deg_gr = \ '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ @@ -199,12 +167,28 @@ def run_cmor_RUN(filename, table, opt_var_name): pass print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') if some_return != 0: - print('didnt pass ocean-file test number 2... exit.') -# sys.exit() + print('didnt pass ocean-file test number 7. exit.') + sys.exit() +# 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) +# atmos, Amon / ch4global +testfile_atmos_scalar_gn_Amon_nolon_nolat = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_scalar/ts/monthly/5yr/' + \ + 'atmos_scalar.197001-197412.ch4global.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 +# pass +print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') +##----------------------------------------------------------------------------------------------------------------------------------------------- +# +# ##### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. ##### for this case, that file is: ##### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ @@ -226,11 +210,61 @@ def run_cmor_RUN(filename, table, opt_var_name): # pass #print_the_outcome(some_return,'ocean_monthly_gn / sos') #if some_return != 0: -# print('didnt pass ocean-file test number 1... exit.') +# print('didnt pass ocean-file test 6... exit.') ## sys.exit() # # # +## 6-alt) +## Result - , last checked VERY carfully +## Note: rm -f alt_Omon_sos_gn_input/*nc; rm -rf tmp/CMIP6; # clean up +## python run_test_file_cases.py; # see initial failure of both this test and the alt test +## python scratchwork_rewrite_sos_ocean_monthly_gn_file.py; # rewrite the input into alt location +## rm -rf tmp/CMIP6; python run_test_file_cases.py; # rerun these cases, 9 still fails, but 9-alt succeeds. +#testfile_ALT_ocean_monthly_gn = \ +# './alt_Omon_sos_gn_input/' + \ +# 'ocean_monthly.002101-002512.sos.nc' +#try: +# some_return = run_cmor_RUN(testfile_ALT_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'ALT ocean_monthly_gn / sos') +#if some_return != 0: +# print('didnt pass ALT ocean-file test 6... exit.') +# sys.exit() +# +# +##----------------------------------------------------------------------------------------------------------------------------------------------- + + + + + +## 7-alt) +## Result - , last checked VERY carfulyl +## Note: rm -f alt_Omon_so_gr_input//*nc; rm -rf tmp/CMIP6; # clean up +## python run_test_file_cases.py; # see initial failure of both this test and the alt test +## python scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py; # rewrite the input into alt location +## rm -rf tmp/CMIP6; python run_test_file_cases.py; # rerun these cases, 9 still fails, but 9-alt succeeds. +#testfile_ALT_ocean_monthly_z_1x1deg_gr = \ +# './alt_Omon_so_gr_input/' + \ +# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' +#try: +# some_return = run_cmor_RUN(testfile_ALT_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'ALT ocean_monthly_z_1x1deg_gr / so') +#if some_return != 0: +# print('didnt pass ALT ocean-file test number 7. exit.') +# sys.exit() + + +##----------------------------------------------------------------------------------------------------------------------------------------------- +# # #### THIS CASE MAY WORK if i rewrite the land file correctly, with the right landuse dimension strings ## 9) FAIL (4 dimensional data with no vertical) @@ -250,3 +284,24 @@ def run_cmor_RUN(filename, table, opt_var_name): #if some_return != 0: # print('didnt pass the land-file test. exit.') # #sys.exit() +# +# +## 9-alt) PASS +## Result - success, last checked Dec 10 10:40 am VERY carfully +## Note: rm -f alt_LUmip_input/*nc; rm -rf tmp/CMIP6; # clean up +## python run_test_file_cases.py; # see initial failure of both this test and the alt test +## python scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py; # rewrite the input into alt location +## rm -rf tmp/CMIP6; python run_test_file_cases.py; # rerun these cases, 9 still fails, but 9-alt succeeds. +#testfile_ALT_LUmip_refined_gr1_Emon_landusedim = \ +# './alt_LUmip_input/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' +#try: +# some_return = run_cmor_RUN(testfile_ALT_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# pass +#print_the_outcome(some_return,'ALT LUmip_refined_gr1_Emon_landusedim / gppLut') +#if some_return != 0: +# print('didnt pass ALT the land-file test. exit.') +# sys.exit() From 83eaaa51ff3d4907c7cc2c7860b27ce2a903c431 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 10 Dec 2024 14:43:54 -0500 Subject: [PATCH 30/36] ... annnnnnnd another one ones gone and another ones gone... --- fre/cmor/cmor_mixer.py | 19 ++++++--- run_test_file_cases.py | 89 ++++++++++++------------------------------ 2 files changed, 38 insertions(+), 70 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 00fad903..3100059d 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -379,14 +379,23 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor_lev = None if lev is not None: print(f'(rewrite_netcdf_file_var) assigning cmor_lev') + if vert_dim.lower() in ["landuse", "plev39", "plev30", "plev19", "plev8", "height2m"]: print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') - cmor_vert_dim_name = vert_dim - if vert_dim.lower() == "landuse": + if vert_dim.lower() != "landuse": + cmor_vert_dim_name = vert_dim + cmor_lev = cmor.axis( cmor_vert_dim_name, + coord_vals = lev[:], units = lev_units ) + else: + landuse_str_list=['primary_and_secondary_land', 'pastures', 'crops', 'urban'] cmor_vert_dim_name = "landUse" # this is why can't we have nice things - print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') - cmor_lev = cmor.axis( cmor_vert_dim_name, - coord_vals = lev[:], units = lev_units ) + cmor_lev = cmor.axis( cmor_vert_dim_name, + coord_vals = np.array( + landuse_str_list, + dtype=f'S{len(landuse_str_list[0])}' ), + units = lev_units ) + + elif vert_dim in ["z_l"]: lev_bnds = create_lev_bnds( bound_these = lev, with_these = ds['z_i'] ) diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 065d1dfd..f2f670a0 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -121,7 +121,6 @@ def run_cmor_RUN(filename, table, opt_var_name): print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') - # 4) SUCCEEDs (no longitude coordinate case) # atmos, AERmonZ / ta # just like #1, but lack longitude @@ -164,7 +163,7 @@ def run_cmor_RUN(filename, table, opt_var_name): except Exception as exc: print(f'exception caught: exc=\n{exc}') some_return=-1 - pass +# pass print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') if some_return != 0: print('didnt pass ocean-file test number 7. exit.') @@ -186,6 +185,29 @@ def run_cmor_RUN(filename, table, opt_var_name): print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') + +# 9) SUCCEEDs (needs coordinate variable axis with character string values) +# land, Emon / gppLut +testfile_LUmip_refined_gr1_Emon_landusedim = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/LUmip_refined/ts/monthly/5yr/' + \ + 'LUmip_refined.185001-185412.gppLut.nc' +try: + some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') +if some_return != 0: + print('didnt pass the land-file test. exit.') + #sys.exit() + + + + + + ##----------------------------------------------------------------------------------------------------------------------------------------------- # # @@ -242,66 +264,3 @@ def run_cmor_RUN(filename, table, opt_var_name): -## 7-alt) -## Result - , last checked VERY carfulyl -## Note: rm -f alt_Omon_so_gr_input//*nc; rm -rf tmp/CMIP6; # clean up -## python run_test_file_cases.py; # see initial failure of both this test and the alt test -## python scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py; # rewrite the input into alt location -## rm -rf tmp/CMIP6; python run_test_file_cases.py; # rerun these cases, 9 still fails, but 9-alt succeeds. -#testfile_ALT_ocean_monthly_z_1x1deg_gr = \ -# './alt_Omon_so_gr_input/' + \ -# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' -#try: -# some_return = run_cmor_RUN(testfile_ALT_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'ALT ocean_monthly_z_1x1deg_gr / so') -#if some_return != 0: -# print('didnt pass ALT ocean-file test number 7. exit.') -# sys.exit() - - -##----------------------------------------------------------------------------------------------------------------------------------------------- -# -# -#### THIS CASE MAY WORK if i rewrite the land file correctly, with the right landuse dimension strings -## 9) FAIL (4 dimensional data with no vertical) -## Result - type error, expecting landuse dimension to be string, not an int flag with interpretation -## Fix - rewrite the file and it's data, mapping it's landuse int flag coord to the desired string values -#testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/LUmip_refined/ts/monthly/5yr/' + \ -# 'LUmip_refined.185001-185412.gppLut.nc' -#try: -# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') -#if some_return != 0: -# print('didnt pass the land-file test. exit.') -# #sys.exit() -# -# -## 9-alt) PASS -## Result - success, last checked Dec 10 10:40 am VERY carfully -## Note: rm -f alt_LUmip_input/*nc; rm -rf tmp/CMIP6; # clean up -## python run_test_file_cases.py; # see initial failure of both this test and the alt test -## python scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py; # rewrite the input into alt location -## rm -rf tmp/CMIP6; python run_test_file_cases.py; # rerun these cases, 9 still fails, but 9-alt succeeds. -#testfile_ALT_LUmip_refined_gr1_Emon_landusedim = \ -# './alt_LUmip_input/' + \ -# 'LUmip_refined.185001-185412.gppLut.nc' -#try: -# some_return = run_cmor_RUN(testfile_ALT_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'ALT LUmip_refined_gr1_Emon_landusedim / gppLut') -#if some_return != 0: -# print('didnt pass ALT the land-file test. exit.') -# sys.exit() From 1cac72b872b4089fd9bbcb7758c920d6637b9430 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 10 Dec 2024 17:02:47 -0500 Subject: [PATCH 31/36] LAST OCEAN case- not going down easy! --- fre/cmor/cmor_mixer.py | 78 +++++++++-- run_test_file_cases.py | 297 ++++++++++++++++++----------------------- 2 files changed, 202 insertions(+), 173 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 3100059d..3268b363 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -33,11 +33,29 @@ def from_dis_gimme_dis(from_dis, gimme_dis): return from_dis[gimme_dis][:].copy() except Exception as exc: print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}' - f' from this: {from_dis} ' +# f' from this: {from_dis} ' f' exc = {exc}' f' returning None!' ) return None +def find_statics_file(bronx_file_path): + print('(find_statics_file) HELLO WORLD!') + #assert type(bronx_file_path) == "" + bronx_file_path_elem=bronx_file_path.split('/') + num_elem=len(bronx_file_path_elem) + print(f'bronx_file_path_elem = {bronx_file_path_elem}') + while bronx_file_path_elem[num_elem-2] != 'pp': + bronx_file_path_elem.pop() + num_elem=num_elem-1 + print(bronx_file_path_elem) + statics_path='/'.join(bronx_file_path_elem) + statics_file=glob.glob(statics_path+'/*static*.nc')[0] + if Path(statics_file).exists(): + return statics_file + else: + return None + + def create_lev_bnds(bound_these = None, with_these = None): the_bnds = None assert len(with_these) == len(bound_these) + 1 @@ -99,6 +117,7 @@ def get_iso_datetimes(var_filenames, iso_datetime_arr = None): if len(iso_datetime_arr) < 1: raise ValueError('(get_iso_datetimes) ERROR: iso_datetime_arr has length 0!') + def check_dataset_for_ocean_grid(ds): ''' checks netCDF4.Dataset ds for ocean grid origin, and throws an error if it finds one. accepts @@ -193,7 +212,8 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, netcdf_file = None, target_var = None, json_exp_config = None, - json_table_config = None):#, tmp_dir = None ): + json_table_config = None, prev_path=None, + ):#, tmp_dir = None ): ''' rewrite the input netcdf file nc_fl containing target_var in a CMIP-compliant manner. accepts six arguments, all required: @@ -221,7 +241,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # ocean grids are not implemented yet. print( '(rewrite_netcdf_file_var) checking input netcdf file for oceangrid condition') uses_ocean_grid = check_dataset_for_ocean_grid(ds) - + if uses_ocean_grid: + print('(rewrite_netcdf_file_var) OH BOY you have a file on the native tripolar grid...\n' + ' ... this is gonna be fun!' ) # try to read what coordinate(s) we're going to be expecting for the variable expected_mip_coord_dims=None @@ -249,7 +271,44 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lon_bnds') lon_bnds = from_dis_gimme_dis( from_dis = ds, gimme_dis = "lon_bnds") - + + # the tripolar grid is designed to reduce distortions in ocean data brought on + # by singularities (poles) being placed in oceans (e.g. the N+S poles of standard sphere grid) + # but, the tripolar grid is complex, so the values stored in the file are a lat/lon *on the tripolar grid* + # in order to get spherical lat/lon, one would need to convert on the fly, but implementing such an inverse is not trivial + # thankfully, the spherical lat/lons tend to already be computed in advance, and stored elsewhere. at GFDL they're in "statics" + statics_file_path = None + if all( [ uses_ocean_grid, + lat is None, + lon is None ] ): + try: + print(f'(rewrite_netcdf_file_var) netcdf_file is {netcdf_file}') + statics_file_path = find_statics_file(prev_path) + print(f'(rewrite_netcdf_file_var) statics_file_path is {statics_file_path}') + except Exception as exc: + print(f'(rewrite_netcdf_file_var) WARNING: pretty sure an ocean statics file is needed, but it could not be found.' + ' moving on and doing my best, but i am probably going to break' ) + raise Exception('EXITING BC STATICS') from exc + print(f"statics file found.") + statics_file_name=Path(statics_file_path).name + put_statics_file_here=str(Path(netcdf_file).parent) + shutil.copy(statics_file_path, put_statics_file_here) + del statics_file_path + statics_file_path = put_statics_file_here + '/' + statics_file_name + print(f'statics file path is now: {statics_file_path}') + + statics_ds=nc.Dataset(statics_file_path, 'r') + lat = statics_ds['geolat'][:].copy() + lon = statics_ds['geolon'][:].copy() + + #print(f' geolat = {lat}') + #assert False + + + + + + # read in time_coords + units print(f'(rewrite_netcdf_file_var) attempting to read coordinate time, and units...') time_coords = from_dis_gimme_dis( from_dis = ds, @@ -318,16 +377,19 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # setup cmor latitude axis if relevant cmor_lat = None - if any( [ lat is None, lat_bnds is None ] ): + if any( [ lat is None ] ): print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') else: print(f'(rewrite_netcdf_file_var) assigning cmor_lat') - cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N") + if lat_bnds is None: + cmor_lat = cmor.axis("latitude", coord_vals = lat[:], units = "degrees_N") + else: + cmor_lat = cmor.axis("latitude", coord_vals = lat[:], cell_bounds = lat_bnds, units = "degrees_N") print(f' DONE assigning cmor_lat') # setup cmor longitude axis if relevant cmor_lon = None - if any( [ lon is None, lon_bnds is None ] ): + if any( [ lon is None ] ): print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') else: print(f'(rewrite_netcdf_file_var) assigning cmor_lon') @@ -600,7 +662,7 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None, nc_file_work , target_var , json_exp_config , - json_table_config ) + json_table_config , nc_fls[i] ) except Exception as exc: raise Exception('(cmorize_target_var_files) problem with rewrite_netcdf_file_var. exc=\n' f' {exc}\n' diff --git a/run_test_file_cases.py b/run_test_file_cases.py index f2f670a0..9adae56a 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -76,191 +76,158 @@ def run_cmor_RUN(filename, table, opt_var_name): -# 1) SUCCEEDs -# land, Lmon, gr1 -testfile_land_gr1_Lmon = \ - '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/land/ts/monthly/5yr/' + \ - 'land.005101-005512.lai.nc' -try: - some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -except: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'land_gr1_Lmon / lai') - - -# 2) SUCCEEDs -# atmos, Amon / cl -testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.196001-196412.cl.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') - - -# 3) SUCCEEDs -# atmos, Amon / mc -testfile_atmos_level_cmip_gr1_Amon_fullL = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.195501-195912.mc.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') - - -# 4) SUCCEEDs (no longitude coordinate case) -# atmos, AERmonZ / ta -# just like #1, but lack longitude -testfile_atmos_gr1_AERmonZ_nolons = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ - 'atmos_plev39_cmip.201001-201412.ta.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') - - -# 5) SUCCEEDs -# ocean, Omon / sos -testfile_ocean_monthly_1x1deg_gr = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_1x1deg.190001-190412.sos.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') - - -# 7) SUCCEEDs -# ocean, Omon / so -testfile_ocean_monthly_z_1x1deg_gr = \ - '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_z_1x1deg.000101-000512.so.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') -if some_return != 0: - print('didnt pass ocean-file test number 7. exit.') - sys.exit() - - -# 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) -# atmos, Amon / ch4global -testfile_atmos_scalar_gn_Amon_nolon_nolat = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_scalar/ts/monthly/5yr/' + \ - 'atmos_scalar.197001-197412.ch4global.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 -# pass -print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - - - -# 9) SUCCEEDs (needs coordinate variable axis with character string values) -# land, Emon / gppLut -testfile_LUmip_refined_gr1_Emon_landusedim = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/LUmip_refined/ts/monthly/5yr/' + \ - 'LUmip_refined.185001-185412.gppLut.nc' -try: - some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - pass -print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') -if some_return != 0: - print('didnt pass the land-file test. exit.') - #sys.exit() - - - - - - -##----------------------------------------------------------------------------------------------------------------------------------------------- +## 1) SUCCEEDs +## land, Lmon, gr1 +#testfile_land_gr1_Lmon = \ +# '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/land/ts/monthly/5yr/' + \ +# 'land.005101-005512.lai.nc' +#try: +# some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') +#except: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'land_gr1_Lmon / lai') +# +# +## 2) SUCCEEDs +## atmos, Amon / cl +#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ +# 'atmos_level_cmip.196001-196412.cl.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') +# +# +## 3) SUCCEEDs +## atmos, Amon / mc +#testfile_atmos_level_cmip_gr1_Amon_fullL = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ +# 'atmos_level_cmip.195501-195912.mc.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') +# +# +## 4) SUCCEEDs (no longitude coordinate case) +## atmos, AERmonZ / ta +## just like #1, but lack longitude +#testfile_atmos_gr1_AERmonZ_nolons = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ +# 'atmos_plev39_cmip.201001-201412.ta.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') # # -##### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. -##### for this case, that file is: -##### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -##### 'pp/ocean_monthly/' + \ -##### 'ocean_monthly.static.nc' -##### and that data is stored under "geolon" and "geolat" consuming dims "x" and "y". -## 6) FAIL +## 5) SUCCEEDs ## ocean, Omon / sos -## Result - error, it wants lat/lon, but only xh, yh coordinates are available -#testfile_ocean_monthly_gn = \ +#testfile_ocean_monthly_1x1deg_gr = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ +# 'ocean_monthly_1x1deg.190001-190412.sos.nc' +#try: +# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') +# +# +## 7) SUCCEEDs +## ocean, Omon / so +#testfile_ocean_monthly_z_1x1deg_gr = \ # '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/ocean_monthly/ts/monthly/5yr/' + \ -# 'ocean_monthly.002101-002512.sos.nc' +# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ +# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' #try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +# some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') #except Exception as exc: # print(f'exception caught: exc=\n{exc}') # some_return=-1 -# pass -#print_the_outcome(some_return,'ocean_monthly_gn / sos') +## pass +#print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') #if some_return != 0: -# print('didnt pass ocean-file test 6... exit.') -## sys.exit() +# print('didnt pass ocean-file test number 7. exit.') +# sys.exit() +# +# +## 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) +## atmos, Amon / ch4global +#testfile_atmos_scalar_gn_Amon_nolon_nolat = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_scalar/ts/monthly/5yr/' + \ +# 'atmos_scalar.197001-197412.ch4global.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +## pass +#print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') # # # -## 6-alt) -## Result - , last checked VERY carfully -## Note: rm -f alt_Omon_sos_gn_input/*nc; rm -rf tmp/CMIP6; # clean up -## python run_test_file_cases.py; # see initial failure of both this test and the alt test -## python scratchwork_rewrite_sos_ocean_monthly_gn_file.py; # rewrite the input into alt location -## rm -rf tmp/CMIP6; python run_test_file_cases.py; # rerun these cases, 9 still fails, but 9-alt succeeds. -#testfile_ALT_ocean_monthly_gn = \ -# './alt_Omon_sos_gn_input/' + \ -# 'ocean_monthly.002101-002512.sos.nc' +## 9) SUCCEEDs (needs coordinate variable axis with character string values) +## land, Emon / gppLut +#testfile_LUmip_refined_gr1_Emon_landusedim = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' #try: -# some_return = run_cmor_RUN(testfile_ALT_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') #except Exception as exc: # print(f'exception caught: exc=\n{exc}') # some_return=-1 # pass -#print_the_outcome(some_return,'ALT ocean_monthly_gn / sos') +#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') #if some_return != 0: -# print('didnt pass ALT ocean-file test 6... exit.') -# sys.exit() -# -# -##----------------------------------------------------------------------------------------------------------------------------------------------- +# print('didnt pass the land-file test. exit.') +# #sys.exit() + +#### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. +#### for this case, that file is: +#### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +#### 'pp/ocean_monthly/' + \ +#### 'ocean_monthly.static.nc' +#### and that data is stored under "geolon" and "geolat" consuming dims "x" and "y". +# 6) FAIL +# ocean, Omon / sos +# Result - error, it wants lat/lon, but only xh, yh coordinates are available +testfile_ocean_monthly_gn = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly/ts/monthly/5yr/' + \ + 'ocean_monthly.002101-002512.sos.nc' +try: + some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + pass +print_the_outcome(some_return,'ocean_monthly_gn / sos') +if some_return != 0: + print('didnt pass ocean-file test 6... exit.') +# sys.exit() From 0d5a490045925eea7762ac0cd3745ec8bf48b49f Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 10 Dec 2024 17:03:26 -0500 Subject: [PATCH 32/36] remove now-irrelevant scratchwork files --- ...k_rewrite_gppLut_LUmip_refined_gr1_file.py | 194 ------------------ ...write_so_ocean_monthly_z_1x1deg_gr_file.py | 190 ----------------- 2 files changed, 384 deletions(-) delete mode 100644 scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py delete mode 100644 scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py diff --git a/scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py b/scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py deleted file mode 100644 index edb78242..00000000 --- a/scratchwork_rewrite_gppLut_LUmip_refined_gr1_file.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python -#import sys -#import os -from pathlib import Path - -import numpy -#from numpy.dtypes import StringDType -from netCDF4 import Dataset #, stringtochar - -''' - in landuse, an int flag, values [0, 1, 2, 3] to be interpreted as: - 0 - psl, "primary_and_secondary_land" char string in CMIP6/7 - 1 - pst, "pastures" char string in CMIP6/7 - 2 - crp, "crops" char string in CMIP6/7 - 3 - urb, "urban" char string in CMIP6/7 -''' -#landuse_str_list=["primary_and_secondary_land", "pastures", "crops", "urban"] -landuse_str_list=[b"primary_and_secondary_land", b"pastures", b"crops", b"urban"] - - -# open netcdf file in append mode? write mode? read? -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/LUmip_refined/ts/monthly/5yr/' + \ -gppLut_fin=Dataset('./tmp/LUmip_refined.185001-185412.gppLut.nc', mode='r') -gppLut_fin_ncattrs=gppLut_fin.__dict__ #dictionary - -# the target data of interest -gppLut_var_data = gppLut_fin.variables['gppLut'][:] -gppLut_var_atts = gppLut_fin.variables['gppLut'].__dict__ - -# coordinate variables, their _bnds, and their identically named dimensions -# coordinate variable == a variable with the same name as a dimension. -# pitfall: an "axis" in netcdf is not analagous to a dimension, overloaded term -bnds_coord_data = gppLut_fin.variables['bnds'][:] -bnds_coord_atts = gppLut_fin.variables['bnds'].__dict__ -bnds_coord_dims = gppLut_fin.dimensions['bnds'].size - -time_coord_data = gppLut_fin.variables['time'][:] -time_coord_atts = gppLut_fin.variables['time'].__dict__ -time_coord_bnds = gppLut_fin.variables['time_bnds'][:] -time_coord_bnds_atts = gppLut_fin.variables['time_bnds'].__dict__ -#time_coord_dims = gppLut_fin.dimensions['time'].size - -landuse_coord_data = gppLut_fin.variables['landuse'][:] -landuse_coord_atts = gppLut_fin.variables['landuse'].__dict__ -landuse_coord_dims = gppLut_fin.dimensions['landuse'].size - -lat_coord_data = gppLut_fin.variables['lat'][:] -lat_coord_atts = gppLut_fin.variables['lat'].__dict__ -lat_coord_bnds = gppLut_fin.variables['lat_bnds'][:] -lat_coord_bnds_atts = gppLut_fin.variables['lat_bnds'].__dict__ -lat_coord_dims = gppLut_fin.dimensions['lat'].size - -lon_coord_data = gppLut_fin.variables['lon'][:] -lon_coord_atts = gppLut_fin.variables['lon'].__dict__ -lon_coord_bnds = gppLut_fin.variables['lon_bnds'][:] -lon_coord_bnds_atts = gppLut_fin.variables['lon_bnds'].__dict__ -lon_coord_dims = gppLut_fin.dimensions['lon'].size - -''' - we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work - recall, a netCDF4 file is, basically, 4 sets of things - attributes, i.e effectively global metadata - groups, i.e. a heirarchy with nesting a lot like directories (older netcdf files only have a root group) - dimensions, i.e. a set of named-integers to define the number of divisions on an axis - variables, i.e. arrays representing data with shapes described by the dimensions in the file -''' - -# open the output file -gppLut_fout=Dataset('./alt_LUmip_input/PLAY_LUmip_refined.185001-185412.gppLut.nc',mode='w') -gppLut_fout.setncatts(gppLut_fin_ncattrs) - -''' - from netCDF4 python API doc, for easy referencing - createDimension(self, - dimname, size=None)... None will imply unlimited -''' -gppLut_fout.createDimension( 'time', - None ) #time_coord_dims -gppLut_fout.createDimension( 'bnds', - bnds_coord_dims ) -# this will be for the landUse coordinate axis entries -gppLut_fout.createDimension( 'landUse', - landuse_coord_dims ) -# set to length of longest string in set --> i think this is OK -gppLut_fout.createDimension( 'str_len', - len("primary_and_secondary_land") ) # see below, landuse_str_list -gppLut_fout.createDimension( 'lat', - lat_coord_dims ) -gppLut_fout.createDimension( 'lon', - lon_coord_dims ) - - - - -''' - from netCDF4 python API doc, for easy referencing - def createVariable(self, - varname, datatype, dimensions=(), + lots others ) -''' -# easy variables first. -# bnds -gppLut_fout.createVariable( 'bnds', gppLut_fin.variables['bnds'].dtype, - dimensions = ( gppLut_fout.dimensions['bnds'] ) ) -gppLut_fout.variables['bnds'][:] = bnds_coord_data -gppLut_fout.variables['bnds'].setncatts( bnds_coord_atts ) - -# time -gppLut_fout.createVariable( 'time', gppLut_fin.variables['time'].dtype, - dimensions = ( gppLut_fout.dimensions['time'] ) ) -gppLut_fout.variables['time'][:] = time_coord_data -gppLut_fout.variables['time'].setncatts( time_coord_atts ) - -# time_bnds -gppLut_fout.createVariable( 'time_bnds', gppLut_fin.variables['time_bnds'].dtype, - fill_value = gppLut_fin.variables['time_bnds']._FillValue, # necessary bc of unlimited + extra limited dim shape? - dimensions = ( gppLut_fout.dimensions['time'], - gppLut_fout.dimensions['bnds'] ) ) -gppLut_fout.variables['time_bnds'][:] = time_coord_bnds -for att in time_coord_bnds_atts: #gppLut_fout.variables['time_bnds'].setncatts( time_coord_bnds_atts ) - if att != '_FillValue': - gppLut_fout.variables['time_bnds'].setncattr( att, time_coord_bnds_atts[att] ) - -# lat -gppLut_fout.createVariable( 'lat', gppLut_fin.variables['lat'].dtype, - dimensions = ( gppLut_fout.dimensions['lat'] ) ) -gppLut_fout.variables['lat'][:] = lat_coord_data -gppLut_fout.variables['lat'].setncatts( lat_coord_atts ) - -# lat_bnds -gppLut_fout.createVariable( 'lat_bnds', gppLut_fin.variables['lat_bnds'].dtype, - dimensions = ( gppLut_fout.dimensions['lat'], - gppLut_fout.dimensions['bnds'] ) ) -gppLut_fout.variables['lat_bnds'][:] = lat_coord_bnds -gppLut_fout.variables['lat_bnds'].setncatts( lat_coord_bnds_atts ) - -# lon -gppLut_fout.createVariable( 'lon', gppLut_fin.variables['lon'].dtype, - dimensions = ( gppLut_fout.dimensions['lon'] ) ) -gppLut_fout.variables['lon'][:] = lon_coord_data -gppLut_fout.variables['lon'].setncatts( lon_coord_atts ) - -# lon_bnds -gppLut_fout.createVariable( 'lon_bnds', gppLut_fin.variables['lon_bnds'].dtype, - dimensions = ( gppLut_fout.dimensions['lon'], - gppLut_fout.dimensions['bnds'] ) ) -gppLut_fout.variables['lon_bnds'][:] = lon_coord_bnds -gppLut_fout.variables['lon_bnds'].setncatts( lon_coord_bnds_atts ) - -# landUse -gppLut_fout.createVariable( 'landUse', 'c', #'S1' - dimensions = ( gppLut_fout.dimensions['landUse'], - gppLut_fout.dimensions['str_len'] ) ) -gppLut_fout.variables['landUse'].setncattr( '_Encoding', 'ascii' ) -# determine numpy array datatype, dep on max string length in landuse_str_list -#landUse_datatype = numpy.dtype([ -gppLut_fout.variables['landUse'][:] = numpy.array(landuse_str_list, dtype= f'S{len(landuse_str_list[0])}')#)# -#landUse_var[:] = numpy.array(landuse_str_list, dtype= f'S{len(landuse_str_list[0])}') - -# fairly sure CMOR will fill this in? -gppLut_fout.variables['landUse'].setncattr( - 'requested_values', ' '.join( - [ landuse_str.decode() for landuse_str in landuse_str_list ] ) - ) -gppLut_fout.variables['landUse'].setncattr("standard_name" , "area_type" ) -gppLut_fout.variables['landUse'].setncattr("long_name" , "Land use type" ) -gppLut_fout.variables['landUse'].setncattr("must_have_bounds" , "no" ) -gppLut_fout.variables['landUse'].setncattr("out_name" , 'landuse' ) - - - -# data time!!! -gppLut_fout.createVariable( 'gppLut', gppLut_fin.variables['gppLut'].dtype, - fill_value = gppLut_fin.variables['gppLut']._FillValue, - dimensions = ( gppLut_fout.dimensions['time'], - gppLut_fout.dimensions['landUse'], - gppLut_fout.dimensions['lat'], - gppLut_fout.dimensions['lon'] ) ) -gppLut_fout.variables['gppLut'][:] = gppLut_var_data -for att in gppLut_var_atts: - if att not in ["time_avg_info", "_FillValue"]: - gppLut_fout.variables['gppLut'].setncattr(att, gppLut_var_atts[att] ) - -gppLut_fout.close() - -## test that the two are equivalent "quickly"... -#unmsk_gppLut_var_data=gppLut_var_data[~gppLut_var_data.mask] -#unmsk_gppLut_var_out=gppLut_var_out[~gppLut_var_out.mask] -#for i in range(0, len( unmsk_gppLut_var_data ) ): -# if i%100 == 0: -# print(f'i = {i}') -# diff = unmsk_gppLut_var_data[i] - unmsk_gppLut_var_out[i] -# if diff > 0.: -# print(f'diff = \n {diff}') diff --git a/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py b/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py deleted file mode 100644 index 72ee48e0..00000000 --- a/scratchwork_rewrite_so_ocean_monthly_z_1x1deg_gr_file.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python -#import sys -#import os -from pathlib import Path - -import numpy -#from numpy.dtypes import StringDType -from netCDF4 import Dataset #, stringtochar - - -## we can see from -# ncdump -v lev /data_cmip6/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/piControl/r1i1p1f1/Omon/so/gr/v20180701/so_Omon_GFDL-ESM4_piControl_r1i1p1f1_gr_000101-002012.nc | grep -A 10 'lev = ' -# ncdump -v lev /data_cmip6/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/piControl/r1i1p1f1/Omon/so/gn/v20180701/so_Omon_GFDL-ESM4_piControl_r1i1p1f1_gn_000101-002012.nc | grep -A 10 'lev = ' -# ncdump -v lev /data_cmip6/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/piControl/r1i1p1f1/Omon/so/gn/v20180701/so_Omon_GFDL-ESM4_piControl_r1i1p1f1_gn_000101-002012.nc | grep -A 10 'lev(' -# ncdump -v lev /data_cmip6/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/piControl/r1i1p1f1/Omon/so/gr/v20180701/so_Omon_GFDL-ESM4_piControl_r1i1p1f1_gr_000101-002012.nc | grep -A 10 'lev(' -# that 'z_l' simply needs to be renamed, effectively. - -# open netcdf file in append mode? write mode? read? -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ -so_gr_fin=Dataset('./tmp/ocean_monthly_z_1x1deg.000101-000512.so.nc', mode='r') -so_gr_fin_ncattrs=so_gr_fin.__dict__ #dictionary - -# the target data of interest -so_gr_var_data = so_gr_fin.variables['so'][:] -so_gr_var_atts = so_gr_fin.variables['so'].__dict__ - -# coordinate variables, their _bnds, and their identically named dimensions -# coordinate variable == a variable with the same name as a dimension. -# pitfall: an "axis" in netcdf is not analagous to a dimension, overloaded term - -# N/A #bnds_coord_data = so_gr_fin.variables['bnds'][:] -# N/A #bnds_coord_atts = so_gr_fin.variables['bnds'].__dict__ -bnds_coord_dims = so_gr_fin.dimensions['bnds'].size - -time_coord_data = so_gr_fin.variables['time'][:] -time_coord_atts = so_gr_fin.variables['time'].__dict__ -time_coord_bnds = so_gr_fin.variables['time_bnds'][:] -time_coord_bnds_atts = so_gr_fin.variables['time_bnds'].__dict__ -#time_coord_dims = so_gr_fin.dimensions['time'].size - -lat_coord_data = so_gr_fin.variables['lat'][:] -lat_coord_atts = so_gr_fin.variables['lat'].__dict__ -lat_coord_bnds = so_gr_fin.variables['lat_bnds'][:] -lat_coord_bnds_atts = so_gr_fin.variables['lat_bnds'].__dict__ -lat_coord_dims = so_gr_fin.dimensions['lat'].size - -lon_coord_data = so_gr_fin.variables['lon'][:] -lon_coord_atts = so_gr_fin.variables['lon'].__dict__ -lon_coord_bnds = so_gr_fin.variables['lon_bnds'][:] -lon_coord_bnds_atts = so_gr_fin.variables['lon_bnds'].__dict__ -lon_coord_dims = so_gr_fin.dimensions['lon'].size - -z_l_coord_data = so_gr_fin.variables['z_l'][:] -z_l_coord_atts = so_gr_fin.variables['z_l'].__dict__ -# N/A #z_l_coord_bnds = so_gr_fin.variables['lon_bnds'][:] -# N/A #z_l_coord_bnds_atts = so_gr_fin.variables['lon_bnds'].__dict__ -z_l_coord_dims = so_gr_fin.dimensions['z_l'].size - - - - -''' - we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work - recall, a netCDF4 file is, basically, 4 sets of things - attributes, i.e effectively global metadata - groups, i.e. a heirarchy with nesting a lot like directories (older netcdf files only have a root group) - dimensions, i.e. a set of named-integers to define the number of divisions on an axis - variables, i.e. arrays representing data with shapes described by the dimensions in the file -''' - -# open the output file -so_gr_fout=Dataset('./alt_Omon_so_gr_input/PLAY_ocean_monthly_z_1x1deg.000101-000512.so.nc',mode='w') -so_gr_fout.setncatts(so_gr_fin_ncattrs) - -''' - from netCDF4 python API doc, for easy referencing - createDimension(self, - dimname, size=None)... None will imply unlimited -''' -so_gr_fout.createDimension( 'time', - None ) #time_coord_dims -so_gr_fout.createDimension( 'bnds', - bnds_coord_dims ) - -so_gr_fout.createDimension( 'lat', - lat_coord_dims ) -so_gr_fout.createDimension( 'lon', - lon_coord_dims ) - -#so_gr_fout.createDimension( 'lev', -so_gr_fout.createDimension( 'olevel', - z_l_coord_dims ) - - - - -''' - from netCDF4 python API doc, for easy referencing - def createVariable(self, - varname, datatype, dimensions=(), + lots others ) -''' -# N/A ## easy variables first. -# N/A ## bnds -# N/A #so_gr_fout.createVariable( 'bnds', so_gr_fin.variables['bnds'].dtype, -# N/A # dimensions = ( so_gr_fout.dimensions['bnds'] ) ) -# N/A #so_gr_fout.variables['bnds'][:] = bnds_coord_data -# N/A #so_gr_fout.variables['bnds'].setncatts( bnds_coord_atts ) - -# time -so_gr_fout.createVariable( 'time', so_gr_fin.variables['time'].dtype, - dimensions = ( so_gr_fout.dimensions['time'] ) ) -so_gr_fout.variables['time'][:] = time_coord_data -so_gr_fout.variables['time'].setncatts( time_coord_atts ) - -# time_bnds -so_gr_fout.createVariable( 'time_bnds', so_gr_fin.variables['time_bnds'].dtype, - # N/A #fill_value = so_gr_fin.variables['time_bnds']._FillValue, # necessary bc of unlimited + extra limited dim shape? - dimensions = ( so_gr_fout.dimensions['time'], - so_gr_fout.dimensions['bnds'] ) ) -so_gr_fout.variables['time_bnds'][:] = time_coord_bnds -for att in time_coord_bnds_atts: #so_gr_fout.variables['time_bnds'].setncatts( time_coord_bnds_atts ) - print("HELLO???") - print(f'att = {att}') - if att != '_FillValue': - so_gr_fout.variables['time_bnds'].setncattr( att, time_coord_bnds_atts[att] ) - -# lat -so_gr_fout.createVariable( 'lat', so_gr_fin.variables['lat'].dtype, - dimensions = ( so_gr_fout.dimensions['lat'] ) ) -so_gr_fout.variables['lat'][:] = lat_coord_data -so_gr_fout.variables['lat'].setncatts( lat_coord_atts ) - -# lat_bnds -so_gr_fout.createVariable( 'lat_bnds', so_gr_fin.variables['lat_bnds'].dtype, - dimensions = ( so_gr_fout.dimensions['lat'], - so_gr_fout.dimensions['bnds'] ) ) -so_gr_fout.variables['lat_bnds'][:] = lat_coord_bnds -so_gr_fout.variables['lat_bnds'].setncatts( lat_coord_bnds_atts ) - -# lon -so_gr_fout.createVariable( 'lon', so_gr_fin.variables['lon'].dtype, - dimensions = ( so_gr_fout.dimensions['lon'] ) ) -so_gr_fout.variables['lon'][:] = lon_coord_data -so_gr_fout.variables['lon'].setncatts( lon_coord_atts ) - -# lon_bnds -so_gr_fout.createVariable( 'lon_bnds', so_gr_fin.variables['lon_bnds'].dtype, - dimensions = ( so_gr_fout.dimensions['lon'], - so_gr_fout.dimensions['bnds'] ) ) -so_gr_fout.variables['lon_bnds'][:] = lon_coord_bnds -so_gr_fout.variables['lon_bnds'].setncatts( lon_coord_bnds_atts ) - - -## lev -#so_gr_fout.createVariable( 'lev', so_gr_fin.variables['z_l'].dtype, -# dimensions = ( so_gr_fout.dimensions['lev'] ) ) -#so_gr_fout.variables['lev'][:] = z_l_coord_data -#so_gr_fout.variables['lev'].setncatts( z_l_coord_atts ) - -# olevel -so_gr_fout.createVariable( 'olevel', so_gr_fin.variables['z_l'].dtype, - dimensions = ( so_gr_fout.dimensions['olevel'] ) ) -so_gr_fout.variables['olevel'][:] = z_l_coord_data -so_gr_fout.variables['olevel'].setncatts( z_l_coord_atts ) - - -# data time!!! -so_gr_fout.createVariable( 'so', so_gr_fin.variables['so'].dtype, - fill_value = so_gr_fin.variables['so']._FillValue, - dimensions = ( so_gr_fout.dimensions['time'], - so_gr_fout.dimensions['olevel'], #so_gr_fout.dimensions['lev'], #TODO SHOULD NOT BE NONE!!!! - so_gr_fout.dimensions['lat'], - so_gr_fout.dimensions['lon'] ) ) -so_gr_fout.variables['so'][:] = so_gr_var_data -for att in so_gr_var_atts: - if att not in ["time_avg_info", "_FillValue"]: - so_gr_fout.variables['so'].setncattr(att, so_gr_var_atts[att] ) - -so_gr_fout.close() - -## test that the two are equivalent "quickly"... -#unmsk_so_gr_var_data=so_gr_var_data[~so_gr_var_data.mask] -#unmsk_so_gr_var_out=so_gr_var_out[~so_gr_var_out.mask] -#for i in range(0, len( unmsk_so_gr_var_data ) ): -# if i%100 == 0: -# print(f'i = {i}') -# diff = unmsk_so_gr_var_data[i] - unmsk_so_gr_var_out[i] -# if diff > 0.: -# print(f'diff = \n {diff}') From 7b364e2cd0634098306362bae85b5cfdcc5d8282 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 12 Dec 2024 16:00:08 -0500 Subject: [PATCH 33/36] the code now finds everything it needs from the statics file and appends it appropriately to the input file. but CMOR is not having it! what am i missing? --- fre/cmor/cmor_mixer.py | 54 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 3268b363..6858718a 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -22,7 +22,6 @@ # ----- \end consts -#### ------ helper functions ------ ### def from_dis_gimme_dis(from_dis, gimme_dis): ''' gives you gimme_dis from from_dis. accepts two arguments, both mandatory. @@ -235,7 +234,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # open the input file print(f"(rewrite_netcdf_file_var) opening {netcdf_file}" ) - ds = nc.Dataset(netcdf_file,'a') + ds = nc.Dataset(netcdf_file,'r+')#'a') # ocean grids are not implemented yet. @@ -288,19 +287,60 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, except Exception as exc: print(f'(rewrite_netcdf_file_var) WARNING: pretty sure an ocean statics file is needed, but it could not be found.' ' moving on and doing my best, but i am probably going to break' ) - raise Exception('EXITING BC STATICS') from exc - print(f"statics file found.") + raise Exception('(rewrite_netcdf_file_var) EXITING BC STATICS') from exc + print(f"(rewrite_netcdf_file_var) statics file found.") statics_file_name=Path(statics_file_path).name put_statics_file_here=str(Path(netcdf_file).parent) shutil.copy(statics_file_path, put_statics_file_here) del statics_file_path statics_file_path = put_statics_file_here + '/' + statics_file_name - print(f'statics file path is now: {statics_file_path}') + print(f'(rewrite_netcdf_file_var) statics file path is now: {statics_file_path}') statics_ds=nc.Dataset(statics_file_path, 'r') - lat = statics_ds['geolat'][:].copy() - lon = statics_ds['geolon'][:].copy() + # grab the lat/lon points, have shape (yh, xh) + statics_lat = from_dis_gimme_dis(statics_ds, 'geolat')#statics_ds['geolat'][:]#.copy() + statics_lon = from_dis_gimme_dis(statics_ds, 'geolon')#statics_ds['geolon'][:]#.copy() + print(f'FOO min entry of geolat: {statics_lat[:].data.min()}') + print(f'BAR min entry of geolon: {statics_lon[:].data.min()}') + + lat = ds.createVariable('lat', np.float32, ('yh', 'xh') ) + lat[:] = statics_lat[:] + lon = ds.createVariable('lon', np.float32, ('yh', 'xh') ) + lon[:] = statics_lon[:] + print(f'FOO min entry of lat: {lat[:].data.min()}') + print(f'BAR min entry of lon: {lon[:].data.min()}') + + # grab the corners of the cells, should have shape (yh+1, xh+1) + lat_c = from_dis_gimme_dis(statics_ds,'geolat_c') + lon_c = from_dis_gimme_dis(statics_ds,'geolon_c') + print(f'FOO min entry of geolat_c: {lat_c[:].data.min()}') + print(f'BAR min entry of geolon_c: {lon_c[:].data.min()}') + + vertex = 4 + ds.createDimension('vertex', vertex) + + lat_bnds = ds.createVariable('lat_bnds', np.float32, ('yh', 'xh', 'vertex') ) + lat_bnds[:,:,0] = lat_c[1:,1:] # NE corner + lat_bnds[:,:,1] = lat_c[1:,:-1] # NW corner + lat_bnds[:,:,2] = lat_c[:-1,:-1] # SW corner + lat_bnds[:,:,3] = lat_c[:-1,1:] # SE corner + + + lon_bnds = ds.createVariable('lon_bnds', np.float32, ('yh', 'xh', 'vertex') ) + lon_bnds[:,:,0] = lon_c[1:,1:] # NE corner + lon_bnds[:,:,1] = lon_c[1:,:-1] # NW corner + lon_bnds[:,:,2] = lon_c[:-1,:-1] # SW corner + lon_bnds[:,:,3] = lon_c[:-1,1:] # SE corner + + #print ('EXITINGEXITINGEXITINGEXITINGEXITINGEXITINGEXITINGEXITINGEXITINGEXITING') + #assert False + + + + + + #print(f' geolat = {lat}') #assert False From b38f25e839a45e31f8661490425c1c3169911f89 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 19 Dec 2024 15:26:24 -0500 Subject: [PATCH 34/36] Update FRE version for 2025 release --- docs/setup.rst | 6 +++--- docs/usage.rst | 2 +- docs/usage/yaml_framework.rst | 6 +++--- docs/what-is-fre.rst | 2 +- fre/tests/test_fre_cli.py | 4 ++-- setup.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/setup.rst b/docs/setup.rst index 7fd26b14..949d8bd7 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -1,14 +1,14 @@ ===== Setup ===== -fre-cli is conda-installable from the “noaa-gfdl” anaconda channel (https://anaconda.org/NOAA-GFDL/fre-cli) +FRE-cli is conda-installable from the “noaa-gfdl” anaconda channel (https://anaconda.org/NOAA-GFDL/fre-cli) and is deployed on GFDL systems as Environment Modules. On GFDL systems ======================== If you are at GFDL (gaea, PP/AN, workstations), you may skip installation:: - module load fre/2024.01 + module load fre/2025.01 fre --help @@ -26,7 +26,7 @@ Once you have conda available, install the latest fre-cli from the NOAA-GFDL ana To install a specific version:: - conda create --name fre-202401 --channel noaa-gfdl --channel conda-forge fre-cli::2024.01 + conda create --name fre-202501 --channel noaa-gfdl --channel conda-forge fre-cli::2025.01 and activate it:: diff --git a/docs/usage.rst b/docs/usage.rst index e49f106b..1ebc0b21 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,7 +1,7 @@ ============= Usage ============= -Using a set of YAML configuration files, ``fre make`` compiles a FMS-based model, and ``fre pp`` postprocesses the history output and runs diagnostic analysis scripts. Please note that model running is not yet supported in FRE 2024; continue to use FRE Bronx frerun. +Using a set of YAML configuration files, ``fre make`` compiles a FMS-based model, and ``fre pp`` postprocesses the history output and runs diagnostic analysis scripts. Please note that model running is not yet supported in FRE 2025; continue to use FRE Bronx frerun. YAML Framework ======================== diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index 3422044a..74422b16 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -1,4 +1,4 @@ -In order to utilize FRE 2024.01 tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with ``[key]: [value]`` fields. +In order to utilize FRE 2025.01 tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with ``[key]: [value]`` fields. Yaml Formatting ---------- @@ -57,10 +57,10 @@ Where each dash indicates a list. .. code-block:: - &version "2024.01" + &version "2025.01" &stem !join [FRE/, *version] -In this example, the reuasble variable ``stem`` will be parsed as ``FRE/2024.01``. +In this example, the reuasble variable ``stem`` will be parsed as ``FRE/2025.01``. Model Yaml ---------- diff --git a/docs/what-is-fre.rst b/docs/what-is-fre.rst index 6e289ff6..b5a31a04 100644 --- a/docs/what-is-fre.rst +++ b/docs/what-is-fre.rst @@ -4,7 +4,7 @@ What is FRE? FRE, the FMS Runtime Environment, is the companion runtime workflow for FMS-based climate and earth system models, and contains scripts and batch job handlers to compile models, run experiments, and postprocess and analyze the output. Developed around 2004 by GFDL's Modeling System Division, FRE was developed primarily in one repository ("fre-commands", https://github.com/NOAA-GFDL/FRE), used subtools in another repository (FRE-NCtools, https://github.com/NOAA-GFDL/fre-nctools), and was deployed using a set of Environment Modules (https://gitlab.gfdl.noaa.gov/fre/modulefiles). Originally, the major releases of FRE were rivers (Arkansas, Bronx) and the minor releases were numbers. In practice, though, the "Bronx" release name was retained and the number has been incremented over the years. e.g. Bronx-23 is the latest release. -Over the last couple years, MSD's workflow team has reengineered the compiling and postprocessing parts of FRE, in a modern python and Cylc-based ecosystem (running experiments is not yet possible with this new FRE; stay tuned). Following a semantic versioning adopted in other FMS repositories, the reengineered FRE is versioned with a year and incrementing two-digit number. e.g. the first release of 2024 is 2024.01, the second 2024.02, and the first release next year will be 2025.01. (Optional minor releases are also available in the scheme; e.g. 2024.01.01 would be the first minor/patch release after 2024.01.) This version is used as tags in FRE repositories and in the corresponding conda (and in the future, container) release, and can be retrieved from ``fre --version``. +Over the last couple years, MSD's workflow team has reengineered the compiling and postprocessing parts of FRE, in a modern python and Cylc-based ecosystem (running experiments is not yet possible with this new FRE; stay tuned). Following a semantic versioning adopted in other FMS repositories, the reengineered FRE is versioned with a year and incrementing two-digit number. e.g. the first release of 2025 is 2025.01, the second 2025.02, etc. (Optional minor releases are also available in the scheme; e.g. 2025.01.01 would be the first minor/patch release after 2025.01.) This version is used as tags in FRE repositories and in the corresponding conda (and in the future, container) release, and can be retrieved from ``fre --version``. fre-cli (this repository) can be considered a successor to the FRE Bronx “fre-commands” repository, which primarily contains user-facing tools and subtools. fre-workflows (https://github.com/NOAA-GFDL/fre-workflows) is a companion repository containing workflow definitions that can be run by the Cylc workflow engine. It contains workflow-specific elements previously in FRE Bronx, and allows flexibility to support multiple and more complex workflows. The two new FRE repositories are versioned with the same approach, and updates will be released together for some time to ensure compatibility. diff --git a/fre/tests/test_fre_cli.py b/fre/tests/test_fre_cli.py index e7bad1d0..d4d758ce 100644 --- a/fre/tests/test_fre_cli.py +++ b/fre/tests/test_fre_cli.py @@ -24,11 +24,11 @@ def test_cli_fre_option_dne(): def test_fre_version(): ''' module import flavor of below cli test ''' - assert '2024.01' == fre.version + assert '2025.01' == fre.version def test_cli_fre_version(): ''' fre --version ''' result = runner.invoke(fre.fre, args='--version') - expected_out = 'fre, version 2024.01' + expected_out = 'fre, version 2025.01' assert all( [ result.exit_code == 0, expected_out in result.stdout.split('\n') ] ) diff --git a/setup.py b/setup.py index 751efcc1..d01e5577 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='fre-cli', - version='2024.01', + version='2025.01', description='Command Line Interface for FRE commands', author='MSD Workflow Team, Bennett Chang, Dana Singh, Chris Blanton', author_email='oar.gfdl.workflow@noaa.gov', From 781e6549fde24798372e854330f45fac17cd32f3 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 19 Dec 2024 15:47:47 -0500 Subject: [PATCH 35/36] push latest code --- fre/cmor/cmor_mixer.py | 113 +++++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 27 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 6858718a..1047ba75 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -271,15 +271,38 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, lon_bnds = from_dis_gimme_dis( from_dis = ds, gimme_dis = "lon_bnds") - # the tripolar grid is designed to reduce distortions in ocean data brought on + # read in time_coords + units + print(f'(rewrite_netcdf_file_var) attempting to read coordinate time, and units...') + time_coords = from_dis_gimme_dis( from_dis = ds, + gimme_dis = 'time' ) + + time_coord_units = ds["time"].units + print(f" time_coord_units = {time_coord_units}") + + # read in time_bnds , if present + print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, time_bnds') + time_bnds = from_dis_gimme_dis( from_dis = ds, + gimme_dis = 'time_bnds' ) + + # read the input variable data, i believe + print(f'(rewrite_netcdf_file_var) attempting to read variable data, {target_var}') + var = from_dis_gimme_dis( from_dis = ds, + gimme_dis = target_var ) + #var = ds[target_var][:] + # the tripolar grid is designed to reduce distortions in ocean data brought on # by singularities (poles) being placed in oceans (e.g. the N+S poles of standard sphere grid) # but, the tripolar grid is complex, so the values stored in the file are a lat/lon *on the tripolar grid* # in order to get spherical lat/lon, one would need to convert on the fly, but implementing such an inverse is not trivial # thankfully, the spherical lat/lons tend to already be computed in advance, and stored elsewhere. at GFDL they're in "statics" + do_special_ocean_file_stuff=all( [ uses_ocean_grid, + lat is None, + lon is None ] ) + statics_file_path = None - if all( [ uses_ocean_grid, - lat is None, - lon is None ] ): + x, y = None, None + i_ind, j_ind = None, None + cmor_grid_id = None + if do_special_ocean_file_stuff: try: print(f'(rewrite_netcdf_file_var) netcdf_file is {netcdf_file}') statics_file_path = find_statics_file(prev_path) @@ -333,8 +356,36 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, lon_bnds[:,:,2] = lon_c[:-1,:-1] # SW corner lon_bnds[:,:,3] = lon_c[:-1,1:] # SE corner - #print ('EXITINGEXITINGEXITINGEXITINGEXITINGEXITINGEXITINGEXITINGEXITINGEXITING') - #assert False + + print(f'(rewrite_netcdf_file_var) HARD PART: creating indices (j_index) from y (yh)') + y = from_dis_gimme_dis(ds, 'yh') + + print(f' ds.createVariable...') + #j_ind = ds.createVariable('j', int, ('yh') ) + j_ind = ds.createVariable('j_index', np.int32, ('yh') ) + print(f' np.arange...') + #j_ind[:] = np.zeros(len(y), dtype=int ) + j_ind[:] = np.arange(0, len(y), dtype=np.int32 ) + + + print(f'(rewrite_netcdf_file_var) HARD PART: creating indices (i_index) from x (xh)') + x = from_dis_gimme_dis(ds, 'xh') + + print(f' ds.createVariable...') + #i_ind = ds.createVariable('i', int, ('xh') ) + i_ind = ds.createVariable('i_index', np.int32, ('xh') ) + print(f' np.arange...') + #i_ind[:] = np.zeros(len(x), dtype=int ) + i_ind[:] = np.arange(0, len(x), dtype=np.int32 ) + + #cmor_grid_id = cmor.grid( ) + + #var.coordinates = 'lat lon' + var.coordinates = 'j_index i_index' + #var.coordinates = '' + + + @@ -349,25 +400,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, - # read in time_coords + units - print(f'(rewrite_netcdf_file_var) attempting to read coordinate time, and units...') - time_coords = from_dis_gimme_dis( from_dis = ds, - gimme_dis = 'time' ) - time_coord_units = ds["time"].units - print(f" time_coord_units = {time_coord_units}") - - # read in time_bnds , if present - print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, time_bnds') - time_bnds = from_dis_gimme_dis( from_dis = ds, - gimme_dis = 'time_bnds' ) - - # read the input variable data, i believe - print(f'(rewrite_netcdf_file_var) attempting to read variable data, {target_var}') - var = from_dis_gimme_dis( from_dis = ds, - gimme_dis = target_var ) - #var = ds[target_var][:] - # grab var_dim var_dim = len(var.shape) @@ -409,7 +442,11 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # load CMOR table print(f"(rewrite_netcdf_file_var) cmor is opening json_table_config = {json_table_config}") - cmor.load_table(json_table_config) + if do_special_ocean_file_stuff: + print("FOOOOOOOOOOOOOOOOOOOOOOO"+ str(Path(json_table_config).parent) + '/CMIP6_grids.json') + cmor.load_table( str(Path(json_table_config).parent) + '/CMIP6_grids.json' ) + else: + cmor.load_table(json_table_config) units = proj_table_vars["variable_entry"] [target_var] ["units"] print(f"(rewrite_netcdf_file_var) units={units}") @@ -417,7 +454,12 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # setup cmor latitude axis if relevant cmor_lat = None - if any( [ lat is None ] ): + if do_special_ocean_file_stuff: + print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for an index!') + #cmor_lat = cmor.axis("j", coord_vals = j_ind[:], units = "1") + cmor_lat = cmor.axis("j_index", coord_vals = j_ind[:], units = "1") + #cmor_lat = cmor.axis("projection_y_coordinate", coord_vals = y[:], units = "degrees") + elif any( [ lat is None ] ): print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') else: print(f'(rewrite_netcdf_file_var) assigning cmor_lat') @@ -429,13 +471,30 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # setup cmor longitude axis if relevant cmor_lon = None - if any( [ lon is None ] ): + if do_special_ocean_file_stuff: + print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for an index!') + #cmor_lon = cmor.axis("i", coord_vals = i_ind[:], units = "1") + cmor_lon = cmor.axis("i_index", coord_vals = i_ind[:], units = "1") + #cmor_lon = cmor.axis("projection_x_coordinate", coord_vals = x[:], units = "degrees") + elif any( [ lon is None ] ): print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') else: print(f'(rewrite_netcdf_file_var) assigning cmor_lon') cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") print(f' DONE assigning cmor_lon') + + # setup the cmor_grid when needed (ocean things, typically) + cmor_grid = None + if do_special_ocean_file_stuff: + cmor_grid = cmor.grid([cmor_lat, cmor_lon], + latitude = lat[:], longitude = lon[:], + latitude_vertices = lat_bnds[:], + longitude_vertices = lon_bnds[:]) + + # load back up the normal table file? + cmor.load_table(json_table_config) + # setup cmor time axis if relevant cmor_time = None print(f'(rewrite_netcdf_file_var) assigning cmor_time') From fa6d74155df0e387145184e2b445dae8f0b0848c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 19 Dec 2024 21:00:55 -0500 Subject: [PATCH 36/36] Update yaml_framework.rst --- docs/usage/yaml_framework.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index 74422b16..444da5f4 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -1,4 +1,4 @@ -In order to utilize FRE 2025.01 tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with ``[key]: [value]`` fields. +In order to utilize these FRE tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with ``[key]: [value]`` fields. Yaml Formatting ---------- @@ -53,14 +53,15 @@ Where each dash indicates a list. *ReusableVariable -7. If the reusable variable must be combined with other strings, the **`!join`** constructor is used. Example: +7. If the reusable variable must be combined with other strings, the **`!join`** constructor is used. Simplified example: .. code-block:: - &version "2025.01" - &stem !join [FRE/, *version] + &name "experiment-name" + ... + pp_dir: !join [/archive/$USER/, *name, /, pp] -In this example, the reuasble variable ``stem`` will be parsed as ``FRE/2025.01``. +In this example, the variable ``pp_dir`` will be parsed as ``/archive/$USER/experiment-name/pp``. Model Yaml ----------