From a376cc230ce1f06a5aa7f1e20736f54cf89c5160 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Fri, 23 Aug 2024 08:38:27 +1000 Subject: [PATCH] minor adjustment to docs, conf files and code after tests --- ACDD_conf.yaml | 2 +- CMIP6_conf.yaml | 4 +++- docs/mopdb_command.rst | 12 ++++++++++-- src/mopdata/update_db.py.txt | 9 ++++++++- src/mopper/mop_utils.py | 11 ++++++++--- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/ACDD_conf.yaml b/ACDD_conf.yaml index f507aeb..a86f373 100755 --- a/ACDD_conf.yaml +++ b/ACDD_conf.yaml @@ -174,4 +174,4 @@ attrs: parent: !!bool false # CMOR will add a tracking_id if you want to define a prefix add here tracking_id_prefix: - comment: "post-processed using ACCESS-MOPPeR v0.6.0 https://doi.org/10.5281/zenodo.10346216" + comment: "post-processed using ACCESS-MOPPeR v1.0.0 https://doi.org/10.5281/zenodo.10346216" diff --git a/CMIP6_conf.yaml b/CMIP6_conf.yaml index 9ae85aa..c421df0 100755 --- a/CMIP6_conf.yaml +++ b/CMIP6_conf.yaml @@ -91,6 +91,8 @@ cmor: # walltime in "hh:mm:ss" walltime: '8:00:00' mode: cmip6 + # conda_env: /bin/activate + conda_env: default # Global attributes: these will be added to each files comment unwanted ones # the labels CMIP6/ACDD indicates which ones are necessary to comply with respective standards @@ -161,4 +163,4 @@ attrs: #CMOR will add a tracking_id if you want to define a prefix add here tracking_id_prefix: Conventions: "CF-1.7 CMIP-6.2" - comment: "post-processed using ACCESS-MOPPeR v0.6.0 https://doi.org/10.5281/zenodo.10346216" + comment: "post-processed using ACCESS-MOPPeR v1.0.0 https://doi.org/10.5281/zenodo.10346216" diff --git a/docs/mopdb_command.rst b/docs/mopdb_command.rst index 5c24196..f60d958 100644 --- a/docs/mopdb_command.rst +++ b/docs/mopdb_command.rst @@ -57,16 +57,19 @@ A user that wants to create a mapping table for another AUS2200 simulation can u Create a mapping file --------------------- -.. code-block:: This can be done by providing the model output path and a pattern to match or directly a varlist file From output path: +.. code-block:: + mopdb template -f -m -v From varlist file: +.. code-block:: + mopdb template -f -v This will create a map_.csv file using, if available, information from the mapping table. @@ -110,20 +113,25 @@ The other groups of records require checking, as either the version or the frequ Create an intake catalogue -------------------------- -.. code-block:: This represents an extra step on top of the mapping, so it can be start directly from an existing mapping or from scratch by providing the model ouptut path and a match. From output path: +.. code-block:: + mopdb intake -f -m -v { -a } From varlist file: +.. code-block:: + mopdb intake -f -fl -v { -a } From mapping file: +.. code-block:: + mopdb intake -f -fl -v { -a } NB the model output path is still needed even when passing an existing mapping or variable list. diff --git a/src/mopdata/update_db.py.txt b/src/mopdata/update_db.py.txt index ddcd6a5..c1cb17d 100644 --- a/src/mopdata/update_db.py.txt +++ b/src/mopdata/update_db.py.txt @@ -37,7 +37,8 @@ def update_map(conn, varid, ctable): """Read mappings for variable from map file and update them in filelist """ - keys = ['frequency','realm','timeshot','calculation', 'positive', 'resample'] + keys = ['frequency','realm','timeshot','calculation', + 'positive', 'resample'] keys2 = {'vin': 'input_vars', 'in_units': 'units'} fname = f"maps/{ctable}.json" with open(fname, 'r') as f: @@ -48,6 +49,12 @@ def update_map(conn, varid, ctable): args = {k: row[k] for k in keys} for k,v in keys2.items(): args[k] = row[v] + if 'datadir' in row.keys(): + paths = row['file_structure'].split() + infile = '' + for x in paths: + infile += f"{row['datadir']}/{x} " + args['infile'] = infile cur = conn.cursor() sql = f"UPDATE filelist SET" for k,v in args.items(): diff --git a/src/mopper/mop_utils.py b/src/mopper/mop_utils.py index 12ef93b..3577e25 100755 --- a/src/mopper/mop_utils.py +++ b/src/mopper/mop_utils.py @@ -564,9 +564,13 @@ def get_coords(ctx, ovar, coords): var_log = logging.getLogger(ctx.obj['var_log']) # open ancil grid file to read vertices #PP be careful this is currently hardcoded which is not ok! - ancil_file = ctx.obj[f"grid_{ctx.obj['realm']}"] + ancil_dir = ctx.obj.get('ancils_path', '') + ancil_file = ancil_dir + "/" + ctx.obj.get(f"grid_{ctx.obj['realm']}", '') + if ancil_file == '' or not Path(ancil_file).exists(): + var_log.error(f"Ancil file {ancil_file} not set or inexistent") + sys.exit() var_log.debug(f"getting lat/lon and bnds from ancil file: {ancil_file}") - ds = xr.open_dataset(f"{ctx.obj['ancils_path']}/{ancil_file}") + ds = xr.open_dataset(ancil_file) var_log.debug(f"ancil ds: {ds}") # read lat/lon and vertices mapping cfile = import_files('mopdata').joinpath('latlon_vertices.yaml') @@ -793,8 +797,9 @@ def get_bounds_values(ctx, ds, bname): calc = False var_log = logging.getLogger(ctx.obj['var_log']) var_log.debug(f"Getting bounds values for {bname}") - ancil_file = ctx.obj[f"grid_{ctx.obj['realm']}"] + ancil_file = ctx.obj.get(f"grid_{ctx.obj['realm']}", '') if bname in ds.variables: + var_log.debug(f"Bounds for {bname} in file") bnds_val = ds[bname].values elif ancil_file != "": fname = f"{ctx.obj['ancils_path']}/{ancil_file}"