diff --git a/examples/process_gridded_data/control_time_chunking_with_gridded_data.yaml b/examples/process_gridded_data/control_time_chunking_with_gridded_data.yaml deleted file mode 100644 index 618a23de..00000000 --- a/examples/process_gridded_data/control_time_chunking_with_gridded_data.yaml +++ /dev/null @@ -1,44 +0,0 @@ -analysis: - start_time: '2020-01-01' - end_time: '2020-12-31' - time_interval: 'MS' - output_dir: $HOME/Plots - debug: True - regrid: False - target_grid: $HOME/Data/Grids/cam_grid.nc - time_chunking_with_gridded_data: True - -obs: - - MOD08_M3: - data_format: gridded_eos - datadir: $HOME/Data/MOD08_M3 - obs_type: gridded_data - filename: MOD08_M3.AYYYYDDD.061.*_regrid.nc - regrid: - base_grid: $HOME/Data/Grids/modis_l3_grid.nc - method: bilinear - variables: - AOD_550_Dark_Target_Deep_Blue_Combined_Mean_Mean: - fillvalue: -9999 - scale: 0.001 - units: none - -model: - - MERRA2: - data_format: netcdf - mod_type: merra2 - datadir: $HOME/Data/MERRA2 - files: MERRA2_*.tavgM_2d_aer_Nx.YYYYMM_MM_TOTEXTTAU_regrid.nc4 - regrid: - base_grid: $HOME/Data/Grids/merra2_grid.nc - method: bilinear - variables: - fillvalue: 1.e+15 - scale: 1.0 - units: none - mapping: - MOD08_M3: - TOTEXTTAU: AOD_550_Dark_Target_Deep_Blue_Combined_Mean_Mean - diff --git a/examples/process_gridded_data/process_time_chunking_with_gridded_data.py b/examples/process_gridded_data/process_time_chunking_with_gridded_data.py deleted file mode 100644 index 9224a652..00000000 --- a/examples/process_gridded_data/process_time_chunking_with_gridded_data.py +++ /dev/null @@ -1,20 +0,0 @@ -from melodies_monet import driver - -an = driver.analysis() -an.control = 'control_time_chunking_with_gridded_data.yaml' -an.read_control() -an.setup_regridders() - -for time_interval in an.time_intervals: - - print(time_interval) - - an.open_obs(time_interval=time_interval) - an.open_models(time_interval=time_interval) - - print(an.obs) - for obs in an.obs: - print(an.obs[obs].obj) - print(an.models) - for mod in an.models: - print(an.models[mod].obj) diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index 2101d493..dde7b87c 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -159,48 +159,33 @@ def open_obs(self, time_interval=None, control_dict=None): from .util import analysis_util from .util import read_grid_util - time_chunking_with_gridded_data \ - = 'time_chunking_with_gridded_data' in control_dict['analysis'].keys() \ - and control_dict['analysis']['time_chunking_with_gridded_data'] - - if time_chunking_with_gridded_data: - date_str = time_interval[0].strftime('%Y-%m-%b-%d-%j') - print('obs time chunk %s' % date_str) - obs_vars = analysis_util.get_obs_vars(control_dict) - print(obs_vars) - obs_datasets, filenames = read_grid_util.read_grid_obs( - control_dict, obs_vars, date_str, obs=self.obs) - print(filenames) - self.obj = obs_datasets[self.obs] - + if self.file.startswith("example:"): + example_id = ":".join(s.strip() for s in self.file.split(":")[1:]) + files = [tutorial.fetch_example(example_id)] else: - if self.file.startswith("example:"): - example_id = ":".join(s.strip() for s in self.file.split(":")[1:]) - files = [tutorial.fetch_example(example_id)] - else: - files = sort(glob(self.file)) + files = sort(glob(self.file)) - assert len(files) >= 1, "need at least one" + assert len(files) >= 1, "need at least one" - _, extension = os.path.splitext(files[0]) - try: - if extension in {'.nc', '.ncf', '.netcdf', '.nc4'}: - if len(files) > 1: - self.obj = xr.open_mfdataset(files) - else: - self.obj = xr.open_dataset(files[0]) - elif extension in ['.ict', '.icartt']: - assert len(files) == 1, "monetio.icartt.add_data can only read one file" - self.obj = mio.icartt.add_data(files[0]) - elif extension in ['.csv']: - from .util.read_util import read_aircraft_obs_csv - assert len(files) == 1, "MELODIES-MONET can only read one csv file" - self.obj = read_aircraft_obs_csv(filename=files[0],time_var=self.time_var) + _, extension = os.path.splitext(files[0]) + try: + if extension in {'.nc', '.ncf', '.netcdf', '.nc4'}: + if len(files) > 1: + self.obj = xr.open_mfdataset(files) else: - raise ValueError(f'extension {extension!r} currently unsupported') - except Exception as e: - print('something happened opening file:', e) - return + self.obj = xr.open_dataset(files[0]) + elif extension in ['.ict', '.icartt']: + assert len(files) == 1, "monetio.icartt.add_data can only read one file" + self.obj = mio.icartt.add_data(files[0]) + elif extension in ['.csv']: + from .util.read_util import read_aircraft_obs_csv + assert len(files) == 1, "MELODIES-MONET can only read one csv file" + self.obj = read_aircraft_obs_csv(filename=files[0],time_var=self.time_var) + else: + raise ValueError(f'extension {extension!r} currently unsupported') + except Exception as e: + print('something happened opening file:', e) + return self.add_coordinates_ground() # If ground site then add coordinates based on yaml if necessary self.mask_and_scale() # mask and scale values from the control values @@ -530,10 +515,6 @@ def open_model_files(self, time_interval=None, control_dict=None): print(self.model.lower()) - time_chunking_with_gridded_data \ - = 'time_chunking_with_gridded_data' in control_dict['analysis'].keys() \ - and control_dict['analysis']['time_chunking_with_gridded_data'] - self.glob_files() # Calculate species to input into MONET, so works for all mechanisms in wrfchem # I want to expand this for the other models too when add aircraft data. @@ -550,80 +531,73 @@ def open_model_files(self, time_interval=None, control_dict=None): list_input_var = list_input_var + list(set(self.mapping[obs_map].keys()) - set(list_input_var)) #Only certain models need this option for speeding up i/o. - if time_chunking_with_gridded_data: - date_str = time_interval[0].strftime('%Y-%m-%b-%d-%j') - print('model time chunk %s' % date_str) - model_datasets, filenames = read_grid_util.read_grid_models( - control_dict, date_str, model=self.label) - print(filenames) - self.obj = model_datasets[self.label] - else: - if 'cmaq' in self.model.lower(): - print('**** Reading CMAQ model output...') - self.mod_kwargs.update({'var_list' : list_input_var}) - if self.files_vert is not None: - self.mod_kwargs.update({'fname_vert' : self.files_vert}) - if self.files_surf is not None: - self.mod_kwargs.update({'fname_surf' : self.files_surf}) - if len(self.files) > 1: - self.mod_kwargs.update({'concatenate_forecasts' : True}) - self.obj = mio.models._cmaq_mm.open_mfdataset(self.files,**self.mod_kwargs) - elif 'wrfchem' in self.model.lower(): - print('**** Reading WRF-Chem model output...') - self.mod_kwargs.update({'var_list' : list_input_var}) - self.obj = mio.models._wrfchem_mm.open_mfdataset(self.files,**self.mod_kwargs) - elif 'rrfs' in self.model.lower(): - print('**** Reading RRFS-CMAQ model output...') - if self.files_pm25 is not None: - self.mod_kwargs.update({'fname_pm25' : self.files_pm25}) - self.mod_kwargs.update({'var_list' : list_input_var}) - self.obj = mio.models._rrfs_cmaq_mm.open_mfdataset(self.files,**self.mod_kwargs) - elif 'gsdchem' in self.model.lower(): - print('**** Reading GSD-Chem model output...') - if len(self.files) > 1: - self.obj = mio.fv3chem.open_mfdataset(self.files,**self.mod_kwargs) - else: - self.obj = mio.fv3chem.open_dataset(self.files,**self.mod_kwargs) - elif 'cesm_fv' in self.model.lower(): - print('**** Reading CESM FV model output...') - self.mod_kwargs.update({'var_list' : list_input_var}) - self.obj = mio.models._cesm_fv_mm.open_mfdataset(self.files,**self.mod_kwargs) - # CAM-chem-SE grid or MUSICAv0 - elif 'cesm_se' in self.model.lower(): - print('**** Reading CESM SE model output...') - self.mod_kwargs.update({'var_list' : list_input_var}) - if self.scrip_file.startswith("example:"): - from . import tutorial - example_id = ":".join(s.strip() for s in self.scrip_file.split(":")[1:]) - self.scrip_file = tutorial.fetch_example(example_id) - self.mod_kwargs.update({'scrip_file' : self.scrip_file}) - self.obj = mio.models._cesm_se_mm.open_mfdataset(self.files,**self.mod_kwargs) - #self.obj, self.obj_scrip = read_cesm_se.open_mfdataset(self.files,**self.mod_kwargs) - #self.obj.monet.scrip = self.obj_scrip - elif "camx" in self.model.lower(): - self.mod_kwargs.update({"var_list": list_input_var}) - self.mod_kwargs.update({"surf_only": control_dict['model'][self.label].get('surf_only', False)}) - self.mod_kwargs.update({"fname_met_3D": control_dict['model'][self.label].get('files_vert', None)}) - self.mod_kwargs.update({"fname_met_2D": control_dict['model'][self.label].get('files_met_surf', None)}) - self.obj = mio.models._camx_mm.open_mfdataset(self.files, **self.mod_kwargs) - elif 'raqms' in self.model.lower(): - if time_interval is not None: - # fill filelist with subset - print('subsetting model files to interval') - file_list = tsub.subset_model_filelist(self.files,'%m_%d_%Y_%HZ','6H',time_interval) - else: - file_list = self.files - if len(file_list) > 1: - self.obj = mio.models.raqms.open_mfdataset(file_list,**self.mod_kwargs) - else: - self.obj = mio.models.raqms.open_dataset(file_list) + if 'cmaq' in self.model.lower(): + print('**** Reading CMAQ model output...') + self.mod_kwargs.update({'var_list' : list_input_var}) + if self.files_vert is not None: + self.mod_kwargs.update({'fname_vert' : self.files_vert}) + if self.files_surf is not None: + self.mod_kwargs.update({'fname_surf' : self.files_surf}) + if len(self.files) > 1: + self.mod_kwargs.update({'concatenate_forecasts' : True}) + self.obj = mio.models._cmaq_mm.open_mfdataset(self.files,**self.mod_kwargs) + elif 'wrfchem' in self.model.lower(): + print('**** Reading WRF-Chem model output...') + self.mod_kwargs.update({'var_list' : list_input_var}) + self.obj = mio.models._wrfchem_mm.open_mfdataset(self.files,**self.mod_kwargs) + elif 'rrfs' in self.model.lower(): + print('**** Reading RRFS-CMAQ model output...') + if self.files_pm25 is not None: + self.mod_kwargs.update({'fname_pm25' : self.files_pm25}) + self.mod_kwargs.update({'var_list' : list_input_var}) + self.obj = mio.models._rrfs_cmaq_mm.open_mfdataset(self.files,**self.mod_kwargs) + elif 'gsdchem' in self.model.lower(): + print('**** Reading GSD-Chem model output...') + if len(self.files) > 1: + self.obj = mio.fv3chem.open_mfdataset(self.files,**self.mod_kwargs) + else: + self.obj = mio.fv3chem.open_dataset(self.files,**self.mod_kwargs) + elif 'cesm_fv' in self.model.lower(): + print('**** Reading CESM FV model output...') + self.mod_kwargs.update({'var_list' : list_input_var}) + self.obj = mio.models._cesm_fv_mm.open_mfdataset(self.files,**self.mod_kwargs) + # CAM-chem-SE grid or MUSICAv0 + elif 'cesm_se' in self.model.lower(): + print('**** Reading CESM SE model output...') + self.mod_kwargs.update({'var_list' : list_input_var}) + if self.scrip_file.startswith("example:"): + from . import tutorial + example_id = ":".join(s.strip() for s in self.scrip_file.split(":")[1:]) + self.scrip_file = tutorial.fetch_example(example_id) + self.mod_kwargs.update({'scrip_file' : self.scrip_file}) + self.obj = mio.models._cesm_se_mm.open_mfdataset(self.files,**self.mod_kwargs) + #self.obj, self.obj_scrip = read_cesm_se.open_mfdataset(self.files,**self.mod_kwargs) + #self.obj.monet.scrip = self.obj_scrip + elif "camx" in self.model.lower(): + self.mod_kwargs.update({"var_list": list_input_var}) + self.mod_kwargs.update({"surf_only": control_dict['model'][self.label].get('surf_only', False)}) + self.mod_kwargs.update({"fname_met_3D": control_dict['model'][self.label].get('files_vert', None)}) + self.mod_kwargs.update({"fname_met_2D": control_dict['model'][self.label].get('files_met_surf', None)}) + self.obj = mio.models._camx_mm.open_mfdataset(self.files, **self.mod_kwargs) + elif 'raqms' in self.model.lower(): + if time_interval is not None: + # fill filelist with subset + print('subsetting model files to interval') + file_list = tsub.subset_model_filelist(self.files,'%m_%d_%Y_%HZ','6H',time_interval) + else: + file_list = self.files + if len(file_list) > 1: + self.obj = mio.models.raqms.open_mfdataset(file_list,**self.mod_kwargs) + else: + self.obj = mio.models.raqms.open_dataset(file_list) + else: + print('**** Reading Unspecified model output. Take Caution...') + if len(self.files) > 1: + self.obj = xr.open_mfdataset(self.files,**self.mod_kwargs) else: - print('**** Reading Unspecified model output. Take Caution...') - if len(self.files) > 1: - self.obj = xr.open_mfdataset(self.files,**self.mod_kwargs) - else: - self.obj = xr.open_dataset(self.files[0],**self.mod_kwargs) + self.obj = xr.open_dataset(self.files[0],**self.mod_kwargs) + self.mask_and_scale() self.rename_vars() # rename any variables as necessary self.sum_variables() @@ -728,7 +702,6 @@ def __init__(self): self.debug = False self.save = None self.read = None - self.time_chunking_with_gridded_data = False # Default to False self.regrid = False # Default to False self.target_grid = None self.obs_regridders = None @@ -814,9 +787,6 @@ def read_control(self, control=None): if 'add_logo' in self.control_dict['analysis'].keys(): self.add_logo = self.control_dict['analysis']['add_logo'] - # set time_chunking_with_gridded_data option, regrid option, and target_grid - if 'time_chunking_with_gridded_data' in self.control_dict['analysis'].keys(): - self.time_chunking_with_gridded_data = self.control_dict['analysis']['time_chunking_with_gridded_data'] if 'regrid' in self.control_dict['analysis'].keys(): self.regrid = self.control_dict['analysis']['regrid'] if 'target_grid' in self.control_dict['analysis'].keys():