Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clean up of driver, removing time_chunking_with_gridded_data option and code blocks #323

Draft
wants to merge 2 commits into
base: develop
Choose a base branch
from
Draft

Clean up of driver, removing time_chunking_with_gridded_data option and code blocks #323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
48f9337
Removed time_chunking_with_gridded_data blocks.
davidfillmore Dec 16, 2024
bdff437
Removed time_chunking_with_gridded_data yaml and example script.
davidfillmore Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 0 additions & 44 deletions examples/process_gridded_data/control_time_chunking_with_gridded_data.yaml
View file
Open in desktop

This file was deleted.

20 changes: 0 additions & 20 deletions examples/process_gridded_data/process_time_chunking_with_gridded_data.py
View file
Open in desktop

This file was deleted.

206 changes: 88 additions & 118 deletions melodies_monet/driver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,48 +159,33 @@ def open_obs(self, time_interval=None, control_dict=None):
from .util import analysis_util
from .util import read_grid_util

time_chunking_with_gridded_data \
= 'time_chunking_with_gridded_data' in control_dict['analysis'].keys() \
and control_dict['analysis']['time_chunking_with_gridded_data']

if time_chunking_with_gridded_data:
date_str = time_interval[0].strftime('%Y-%m-%b-%d-%j')
print('obs time chunk %s' % date_str)
obs_vars = analysis_util.get_obs_vars(control_dict)
print(obs_vars)
obs_datasets, filenames = read_grid_util.read_grid_obs(
control_dict, obs_vars, date_str, obs=self.obs)
print(filenames)
self.obj = obs_datasets[self.obs]

if self.file.startswith("example:"):
example_id = ":".join(s.strip() for s in self.file.split(":")[1:])
files = [tutorial.fetch_example(example_id)]
else:
if self.file.startswith("example:"):
example_id = ":".join(s.strip() for s in self.file.split(":")[1:])
files = [tutorial.fetch_example(example_id)]
else:
files = sort(glob(self.file))
files = sort(glob(self.file))

assert len(files) >= 1, "need at least one"
assert len(files) >= 1, "need at least one"

_, extension = os.path.splitext(files[0])
try:
if extension in {'.nc', '.ncf', '.netcdf', '.nc4'}:
if len(files) > 1:
self.obj = xr.open_mfdataset(files)
else:
self.obj = xr.open_dataset(files[0])
elif extension in ['.ict', '.icartt']:
assert len(files) == 1, "monetio.icartt.add_data can only read one file"
self.obj = mio.icartt.add_data(files[0])
elif extension in ['.csv']:
from .util.read_util import read_aircraft_obs_csv
assert len(files) == 1, "MELODIES-MONET can only read one csv file"
self.obj = read_aircraft_obs_csv(filename=files[0],time_var=self.time_var)
_, extension = os.path.splitext(files[0])
try:
if extension in {'.nc', '.ncf', '.netcdf', '.nc4'}:
if len(files) > 1:
self.obj = xr.open_mfdataset(files)
else:
raise ValueError(f'extension {extension!r} currently unsupported')
except Exception as e:
print('something happened opening file:', e)
return
self.obj = xr.open_dataset(files[0])
elif extension in ['.ict', '.icartt']:
assert len(files) == 1, "monetio.icartt.add_data can only read one file"
self.obj = mio.icartt.add_data(files[0])
elif extension in ['.csv']:
from .util.read_util import read_aircraft_obs_csv
assert len(files) == 1, "MELODIES-MONET can only read one csv file"
self.obj = read_aircraft_obs_csv(filename=files[0],time_var=self.time_var)
else:
raise ValueError(f'extension {extension!r} currently unsupported')
except Exception as e:
print('something happened opening file:', e)
return

self.add_coordinates_ground() # If ground site then add coordinates based on yaml if necessary
self.mask_and_scale() # mask and scale values from the control values
Expand Down Expand Up @@ -530,10 +515,6 @@ def open_model_files(self, time_interval=None, control_dict=None):

print(self.model.lower())

time_chunking_with_gridded_data \
= 'time_chunking_with_gridded_data' in control_dict['analysis'].keys() \
and control_dict['analysis']['time_chunking_with_gridded_data']

self.glob_files()
# Calculate species to input into MONET, so works for all mechanisms in wrfchem
# I want to expand this for the other models too when add aircraft data.
Expand All @@ -550,80 +531,73 @@ def open_model_files(self, time_interval=None, control_dict=None):
list_input_var = list_input_var + list(set(self.mapping[obs_map].keys()) - set(list_input_var))
#Only certain models need this option for speeding up i/o.

if time_chunking_with_gridded_data:
date_str = time_interval[0].strftime('%Y-%m-%b-%d-%j')
print('model time chunk %s' % date_str)
model_datasets, filenames = read_grid_util.read_grid_models(
control_dict, date_str, model=self.label)
print(filenames)
self.obj = model_datasets[self.label]
else:
if 'cmaq' in self.model.lower():
print('**** Reading CMAQ model output...')
self.mod_kwargs.update({'var_list' : list_input_var})
if self.files_vert is not None:
self.mod_kwargs.update({'fname_vert' : self.files_vert})
if self.files_surf is not None:
self.mod_kwargs.update({'fname_surf' : self.files_surf})
if len(self.files) > 1:
self.mod_kwargs.update({'concatenate_forecasts' : True})
self.obj = mio.models._cmaq_mm.open_mfdataset(self.files,**self.mod_kwargs)
elif 'wrfchem' in self.model.lower():
print('**** Reading WRF-Chem model output...')
self.mod_kwargs.update({'var_list' : list_input_var})
self.obj = mio.models._wrfchem_mm.open_mfdataset(self.files,**self.mod_kwargs)
elif 'rrfs' in self.model.lower():
print('**** Reading RRFS-CMAQ model output...')
if self.files_pm25 is not None:
self.mod_kwargs.update({'fname_pm25' : self.files_pm25})
self.mod_kwargs.update({'var_list' : list_input_var})
self.obj = mio.models._rrfs_cmaq_mm.open_mfdataset(self.files,**self.mod_kwargs)
elif 'gsdchem' in self.model.lower():
print('**** Reading GSD-Chem model output...')
if len(self.files) > 1:
self.obj = mio.fv3chem.open_mfdataset(self.files,**self.mod_kwargs)
else:
self.obj = mio.fv3chem.open_dataset(self.files,**self.mod_kwargs)
elif 'cesm_fv' in self.model.lower():
print('**** Reading CESM FV model output...')
self.mod_kwargs.update({'var_list' : list_input_var})
self.obj = mio.models._cesm_fv_mm.open_mfdataset(self.files,**self.mod_kwargs)
# CAM-chem-SE grid or MUSICAv0
elif 'cesm_se' in self.model.lower():
print('**** Reading CESM SE model output...')
self.mod_kwargs.update({'var_list' : list_input_var})
if self.scrip_file.startswith("example:"):
from . import tutorial
example_id = ":".join(s.strip() for s in self.scrip_file.split(":")[1:])
self.scrip_file = tutorial.fetch_example(example_id)
self.mod_kwargs.update({'scrip_file' : self.scrip_file})
self.obj = mio.models._cesm_se_mm.open_mfdataset(self.files,**self.mod_kwargs)
#self.obj, self.obj_scrip = read_cesm_se.open_mfdataset(self.files,**self.mod_kwargs)
#self.obj.monet.scrip = self.obj_scrip
elif "camx" in self.model.lower():
self.mod_kwargs.update({"var_list": list_input_var})
self.mod_kwargs.update({"surf_only": control_dict['model'][self.label].get('surf_only', False)})
self.mod_kwargs.update({"fname_met_3D": control_dict['model'][self.label].get('files_vert', None)})
self.mod_kwargs.update({"fname_met_2D": control_dict['model'][self.label].get('files_met_surf', None)})
self.obj = mio.models._camx_mm.open_mfdataset(self.files, **self.mod_kwargs)
elif 'raqms' in self.model.lower():
if time_interval is not None:
# fill filelist with subset
print('subsetting model files to interval')
file_list = tsub.subset_model_filelist(self.files,'%m_%d_%Y_%HZ','6H',time_interval)
else:
file_list = self.files
if len(file_list) > 1:
self.obj = mio.models.raqms.open_mfdataset(file_list,**self.mod_kwargs)
else:
self.obj = mio.models.raqms.open_dataset(file_list)
if 'cmaq' in self.model.lower():
print('**** Reading CMAQ model output...')
self.mod_kwargs.update({'var_list' : list_input_var})
if self.files_vert is not None:
self.mod_kwargs.update({'fname_vert' : self.files_vert})
if self.files_surf is not None:
self.mod_kwargs.update({'fname_surf' : self.files_surf})
if len(self.files) > 1:
self.mod_kwargs.update({'concatenate_forecasts' : True})
self.obj = mio.models._cmaq_mm.open_mfdataset(self.files,**self.mod_kwargs)
elif 'wrfchem' in self.model.lower():
print('**** Reading WRF-Chem model output...')
self.mod_kwargs.update({'var_list' : list_input_var})
self.obj = mio.models._wrfchem_mm.open_mfdataset(self.files,**self.mod_kwargs)
elif 'rrfs' in self.model.lower():
print('**** Reading RRFS-CMAQ model output...')
if self.files_pm25 is not None:
self.mod_kwargs.update({'fname_pm25' : self.files_pm25})
self.mod_kwargs.update({'var_list' : list_input_var})
self.obj = mio.models._rrfs_cmaq_mm.open_mfdataset(self.files,**self.mod_kwargs)
elif 'gsdchem' in self.model.lower():
print('**** Reading GSD-Chem model output...')
if len(self.files) > 1:
self.obj = mio.fv3chem.open_mfdataset(self.files,**self.mod_kwargs)
else:
self.obj = mio.fv3chem.open_dataset(self.files,**self.mod_kwargs)
elif 'cesm_fv' in self.model.lower():
print('**** Reading CESM FV model output...')
self.mod_kwargs.update({'var_list' : list_input_var})
self.obj = mio.models._cesm_fv_mm.open_mfdataset(self.files,**self.mod_kwargs)
# CAM-chem-SE grid or MUSICAv0
elif 'cesm_se' in self.model.lower():
print('**** Reading CESM SE model output...')
self.mod_kwargs.update({'var_list' : list_input_var})
if self.scrip_file.startswith("example:"):
from . import tutorial
example_id = ":".join(s.strip() for s in self.scrip_file.split(":")[1:])
self.scrip_file = tutorial.fetch_example(example_id)
self.mod_kwargs.update({'scrip_file' : self.scrip_file})
self.obj = mio.models._cesm_se_mm.open_mfdataset(self.files,**self.mod_kwargs)
#self.obj, self.obj_scrip = read_cesm_se.open_mfdataset(self.files,**self.mod_kwargs)
#self.obj.monet.scrip = self.obj_scrip
elif "camx" in self.model.lower():
self.mod_kwargs.update({"var_list": list_input_var})
self.mod_kwargs.update({"surf_only": control_dict['model'][self.label].get('surf_only', False)})
self.mod_kwargs.update({"fname_met_3D": control_dict['model'][self.label].get('files_vert', None)})
self.mod_kwargs.update({"fname_met_2D": control_dict['model'][self.label].get('files_met_surf', None)})
self.obj = mio.models._camx_mm.open_mfdataset(self.files, **self.mod_kwargs)
elif 'raqms' in self.model.lower():
if time_interval is not None:
# fill filelist with subset
print('subsetting model files to interval')
file_list = tsub.subset_model_filelist(self.files,'%m_%d_%Y_%HZ','6H',time_interval)
else:
file_list = self.files
if len(file_list) > 1:
self.obj = mio.models.raqms.open_mfdataset(file_list,**self.mod_kwargs)
else:
self.obj = mio.models.raqms.open_dataset(file_list)

else:
print('**** Reading Unspecified model output. Take Caution...')
if len(self.files) > 1:
self.obj = xr.open_mfdataset(self.files,**self.mod_kwargs)
else:
print('**** Reading Unspecified model output. Take Caution...')
if len(self.files) > 1:
self.obj = xr.open_mfdataset(self.files,**self.mod_kwargs)
else:
self.obj = xr.open_dataset(self.files[0],**self.mod_kwargs)
self.obj = xr.open_dataset(self.files[0],**self.mod_kwargs)

self.mask_and_scale()
self.rename_vars() # rename any variables as necessary
self.sum_variables()
Expand Down Expand Up @@ -728,7 +702,6 @@ def __init__(self):
self.debug = False
self.save = None
self.read = None
self.time_chunking_with_gridded_data = False # Default to False
self.regrid = False # Default to False
self.target_grid = None
self.obs_regridders = None
Expand Down Expand Up @@ -814,9 +787,6 @@ def read_control(self, control=None):
if 'add_logo' in self.control_dict['analysis'].keys():
self.add_logo = self.control_dict['analysis']['add_logo']

# set time_chunking_with_gridded_data option, regrid option, and target_grid
if 'time_chunking_with_gridded_data' in self.control_dict['analysis'].keys():
self.time_chunking_with_gridded_data = self.control_dict['analysis']['time_chunking_with_gridded_data']
if 'regrid' in self.control_dict['analysis'].keys():
self.regrid = self.control_dict['analysis']['regrid']
if 'target_grid' in self.control_dict['analysis'].keys():
Expand Down
Loading