prototype workflow for #3

ACDtools/ard.py

@@ -6,6 +6,7 @@
 """
 # Standard library imports
 import os
+import datetime
 
 
 # Third-party imports
@@ -260,3 +261,12 @@ def find_chunking_info(catalog_search, var_name, return_results=False):
     else:
         return None
 
+def save_n_drop_multidim_lat_lon(ds, save_coords_dir, coords_name='ACCESS-ESM1.5'):
+    coords = ds[['latitude', 'longitude','vertices_latitude','vertices_longitude']]
+    current_datetime = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"{coords_name}_{current_datetime}_coords.nc"
+    coords.to_netcdf(save_coords_dir + filename)
+    ds_dropped = ds.drop(['latitude','longitude','vertices_latitude','vertices_longitude'])
+    ds_dropped.attrs['coords_filename'] = save_coords_dir + filename
+    ds_dropped.attrs['NOTE on coordinates'] = 'the multidimensional latitude and longitude coordinates have been saved as a separate NetCDF file'
+    return ds_dropped

ACDtools/util.py

@@ -62,7 +62,7 @@ def load_config(config_file='config.yaml'):
     ----------
     config_file : str, optional
         The name or relative path of the YAML configuration file to load. 
-        By default, it looks for 'config.yaml' in the same directory as the script.
+        By default, it looks for 'config.yaml' one directory upstream from the script.
 
     Returns
     -------
@@ -304,6 +304,14 @@ def show_methods(your_object):
     # Print all the methods
     for method in methods_only:
         print(method)
+
+def remove_encoding(DS):
+    for var in DS:
+        DS[var].encoding = {}
+
+    for coord in DS.coords:
+        DS[coord].encoding = {}
+    return DS
 
 
 

job_config.yaml

@@ -0,0 +1,34 @@
+paths:
+  write_dir: '/scratch/es60/ard/models/ACCESS-ESM15/ARD/'
+  save_coords_dir: '/scratch/es60/ard/models/ACCESS-ESM15/ARD/coords/'
+dask_cluster:
+  netcdf_work: # threads per worker set to deal with https://forum.access-hive.org.au/t/netcdf-not-a-valid-id-errors/389
+    n_workers: None  # Number of workers for default cluster
+    threads_per_worker: 1  # threads per worker set to deal with https://forum.access-hive.org.au/t/netcdf-not-a-valid-id-errors/389
+    memory_limit: None  # Default memory limit per worker
+  zarr_work:
+    n_workers: None  # Number of workers for default cluster
+    threads_per_worker: None  # Default threads per worker
+    memory_limit: None  # Default Memory limit per worker
+catalog_search_query_dict:
+  experiment_id: 'piControl'
+  source_id: 'ACCESS-ESM1-5'
+  variable_id: 'thetao'
+  table_id: 'Omon'
+chunking_key:
+  'ACCESS_ESM15_3D'
+chunking:
+  ACCESS_ESM15_2D: #{'chunks':{'member':1,'time':220,'j':300,'i':360}}
+    chunks:
+      member: 1
+      time: 220
+      i: 360
+      j: 300
+  ACCESS_ESM15_3D: #{'chunks':{'member':?,'time':?,'lev':-1,'j':-1,'i':-1}}
+    chunks:
+      member: 1
+      time: 12
+      lev: -1
+      i: -1
+      j: -1
+