From 1451960ff6f416b3c6edb0b3dc505ba5b473811e Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 24 May 2024 11:50:53 -0600 Subject: [PATCH] clean up pb2nc wrapper to be more consistent with other ReformatPoint wrappers --- .../wrappers/pb2nc/test_pb2nc_wrapper.py | 54 ++++-------- metplus/wrappers/pb2nc_wrapper.py | 85 ++++--------------- metplus/wrappers/reformat_point_wrapper.py | 6 +- 3 files changed, 38 insertions(+), 107 deletions(-) diff --git a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py index 91a8a393c..958806cb2 100644 --- a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py @@ -75,38 +75,6 @@ def test_pb2nc_missing_inputs(metplus_config, get_test_data_dir, missing, assert wrapper.errors == errors -# --------------------- -# test_get_command -# test that command is generated correctly -# --------------------- -@pytest.mark.parametrize( - # list of input files - 'infiles', [ - ['file1'], - ['file1', 'file2'], - ['file1', 'file2', 'file3'], - ] -) -@pytest.mark.wrapper -def test_get_command(metplus_config, infiles): - pb = pb2nc_wrapper(metplus_config) - pb.outfile = 'outfilename.txt' - pb.outdir = pb.config.getdir('OUTPUT_BASE') - outpath = os.path.join(pb.outdir, pb.outfile) - pb.infiles = infiles - config_file = pb.c_dict['CONFIG_FILE'] - cmd = pb.get_command() - if not infiles: - expected_cmd = None - else: - expected_cmd = pb.app_path + ' -v 2 ' + infiles[0] + ' ' + outpath + ' ' + config_file - if len(infiles) > 1: - for infile in infiles[1:]: - expected_cmd += ' -pbfile ' + infile - - assert cmd == expected_cmd - - # --------------------- # test_find_input_files # test files can be found with find_input_files with varying offset lists @@ -291,6 +259,14 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), ({'TIME_OFFSET_WARNING': 2, 'PB2NC_TIME_OFFSET_WARNING': 4}, {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), + # 1 extra file + ({'PB2NC_INPUT_TEMPLATE': ('ndas.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr,' + 'another_file.nr')}, {}), + # 2 extra files + ({'PB2NC_INPUT_TEMPLATE': ('ndas.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr,' + 'another_file.nr,yet_another_file.nr')}, + {}), + ] ) @pytest.mark.wrapper @@ -335,20 +311,26 @@ def test_pb2nc_all_fields(metplus_config, config_overrides, env_var_values, config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') + extra_file_args = '' + if 'PB2NC_INPUT_TEMPLATE' in config_overrides: + extra_files = config_overrides['PB2NC_INPUT_TEMPLATE'].split(',')[1:] + for extra_file in extra_files: + extra_file_args += f' -pbfile {input_dir}/{extra_file}' + valid_args = '' if 'PB2NC_VALID_BEGIN' in config_overrides: valid_args += f' -valid_beg {valid_beg}' if 'PB2NC_VALID_END' in config_overrides: valid_args += f' -valid_end {valid_end}' - expected_cmds = [(f"{app_path} {verbosity} " + expected_cmds = [(f"{app_path} " f"{input_dir}/ndas.t00z.prepbufr.tm12.20070401.nr " f"{out_dir}/2007033112.nc " - f"{config_file}{valid_args}"), - (f"{app_path} {verbosity} " + f"{config_file}{extra_file_args}{valid_args} {verbosity}"), + (f"{app_path} " f"{input_dir}/ndas.t12z.prepbufr.tm12.20070401.nr " f"{out_dir}/2007040100.nc " - f"{config_file}{valid_args}"), + f"{config_file}{extra_file_args}{valid_args} {verbosity}"), ] all_cmds = wrapper.run_all_times() diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index 37c468abb..43bcd1615 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -18,7 +18,7 @@ class PB2NCWrapper(ReformatPointWrapper): - """! Wrapper to the MET tool pb2nc which converts prepbufr files + """!Wrapper to the MET tool pb2nc which converts prepbufr files to NetCDF for MET's point_stat tool can recognize. """ RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' @@ -64,11 +64,7 @@ def create_c_dict(self): """! Create a data structure (dictionary) that contains all the values set in the configuration files - Args: - - Returns: - c_dict - a dictionary containing the settings in the - configuration files + @returns dictionary containing the settings from the configuration files """ c_dict = super().create_c_dict() @@ -104,10 +100,8 @@ def create_c_dict(self): self.handle_file_window_variables(c_dict, data_types=['OBS']) - c_dict['VALID_BEGIN_TEMPLATE'] = self.config.getraw('config', - 'PB2NC_VALID_BEGIN') - c_dict['VALID_END_TEMPLATE'] = self.config.getraw('config', - 'PB2NC_VALID_END') + c_dict['VALID_BEG'] = self.config.getraw('config', 'PB2NC_VALID_BEGIN') + c_dict['VALID_END'] = self.config.getraw('config', 'PB2NC_VALID_END') c_dict['ALLOW_MULTIPLE_FILES'] = True @@ -131,67 +125,22 @@ def create_c_dict(self): return c_dict - def set_valid_window_variables(self, time_info): - begin_template = self.c_dict['VALID_BEGIN_TEMPLATE'] - end_template = self.c_dict['VALID_END_TEMPLATE'] - - if begin_template: - self.c_dict['VALID_WINDOW_BEGIN'] = do_string_sub(begin_template, - **time_info) - - if end_template: - self.c_dict['VALID_WINDOW_END'] = do_string_sub(end_template, - **time_info) - - def run_at_time_once(self, time_info): - """!Find files needed to run pb2nc and run if found""" - # look for input files to process - offset_time_info = self.find_input_files(time_info) - if not offset_time_info: - return - - # look for output file path and skip running pb2nc if necessary - if not self.find_and_check_output_file(offset_time_info): - return - - # set environment variables to be passed to MET config file - self.set_environment_variables(offset_time_info) - - self.set_valid_window_variables(offset_time_info) - + def set_command_line_arguments(self, time_info): # handle config file substitution - self.c_dict['CONFIG_FILE'] = do_string_sub(self.c_dict['CONFIG_FILE'], - **offset_time_info) + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) + self.args.append(config_file) - # build and run command - self.build() - - def get_command(self): - """! Builds the command to run the MET application - @rtype string - @return Returns a MET command with arguments that you can run - """ - cmd = f"{self.app_path} -v {self.c_dict['VERBOSITY']}" - - for arg in self.args: - cmd += f' {arg}' - - cmd += f" {self.infiles[0]}" - - out_path = self.get_output_path() - cmd += f" {out_path}" - - cmd += f" {self.c_dict['CONFIG_FILE']}" - - # add additional input files with -pbfile argument + # if more than 2 input files are provided, add them with -pbfile if len(self.infiles) > 1: for infile in self.infiles[1:]: - cmd += f" -pbfile {infile}" - - if self.c_dict.get('VALID_WINDOW_BEGIN'): - cmd += f" -valid_beg {self.c_dict['VALID_WINDOW_BEGIN']}" + self.args.append(f"-pbfile {infile}") - if self.c_dict.get('VALID_WINDOW_END'): - cmd += f" -valid_end {self.c_dict['VALID_WINDOW_END']}" + # reset infiles to only include first file + self.infiles = [self.infiles[0]] - return cmd + for beg_end in ('VALID_BEG', 'VALID_END'): + template = self.c_dict[beg_end] + if not template: + continue + template = do_string_sub(template, **time_info) + self.args.append(f"-{beg_end.lower()} {template}") diff --git a/metplus/wrappers/reformat_point_wrapper.py b/metplus/wrappers/reformat_point_wrapper.py index 1046ecc92..c40dfb098 100755 --- a/metplus/wrappers/reformat_point_wrapper.py +++ b/metplus/wrappers/reformat_point_wrapper.py @@ -23,10 +23,13 @@ class ReformatPointWrapper(RuntimeFreqWrapper): def create_c_dict(self): c_dict = super().create_c_dict() app_upper = self.app_name.upper() + + # populate OBS input templates using app name self.get_input_templates(c_dict, { 'OBS': {'prefix': app_upper, 'required': True}, }) + # set output templates using app name c_dict['OUTPUT_DIR'] = self.config.getdir(f'{app_upper}_OUTPUT_DIR', '') c_dict['OUTPUT_TEMPLATE'] = ( self.config.getraw('config', f'{app_upper}_OUTPUT_TEMPLATE') @@ -45,9 +48,6 @@ def get_command(self): f" {self.get_output_path()}" f"{' ' + ' '.join(self.args) if self.args else ''}" f" -v {self.c_dict['VERBOSITY']}") - # return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" - # f" {' '.join(self.infiles)} {self.get_output_path()}" - # f" {' '.join(self.args)}".rstrip()) def _get_offset_time_info(self, time_info): """!Get offset value that was used to find input data so the output