add create intertable with api test #4

src/pyg2p/main/api.py

@@ -114,12 +114,17 @@ def _define_exec_params(self):
         self._vars['download_configuration'] = self.api_conf.get('downloadConf')
         self._vars['under_api'] = True
         self._vars['check_conf'] = self.api_conf.get('checkConf')
+
+        # INTERTABLES and GEOPOTENTIALS paths handling
         user_intertables = self._vars['interpolation.dir'] or self.configuration.default_interpol_dir
         user_geopotentials = self._vars['geopotential.dir'] or self.configuration.default_geopotential_dir
+        if not self.configuration.intertables.data_path:
+            self.configuration.intertables.data_path = user_intertables
         self._vars['interpolation.dirs'] = {'global': self.configuration.intertables.global_data_path,
                                             'user': user_intertables}
         self._vars['geopotential.dirs'] = {'global': self.configuration.geopotentials.global_data_path,
                                            'user': user_geopotentials}
+
         self.is_config_command = self.to_add_geopotential or self.to_download_conf or self.to_check_conf
         self._vars['execution.doAggregation'] = False
         self._vars['execution.doConversion'] = False

src/pyg2p/main/config.py

@@ -266,6 +266,7 @@ def get_filepath(self, grid_id, additional=None):
                 NO_FILE_GEOPOTENTIAL,
                 details=f'id:{grid_id}, filename:{filename}, Searched in: {self.data_path}, {self.global_data_path}, {additional}'
             )
+        self._log(f'USING GEOFILE {path}', 'INFO')
         return path
 
 

src/pyg2p/main/manipulation/correction.py

@@ -17,6 +17,9 @@ class Corrector(Loggable):
 
     instances = {}
 
+    def __repr__(self):
+        return f'Corrector<{self.grid_id} {self.geo_file}>'
+
     @classmethod
     def get_instance(cls, ctx, grid_id):
         geo_file_ = ctx.geo_file(grid_id)
@@ -29,8 +32,9 @@ def get_instance(cls, ctx, grid_id):
             cls.instances[key] = instance
             return instance
 
-    def __init__(self, ctx, grid_id, geo_file_):
+    def __init__(self, ctx, grid_id, geo_file):
         super().__init__()
+        self.geo_file = geo_file
         self.grid_id = grid_id
         dem_map = ctx.get('correction.demMap')
         self._dem_missing_value, self._dem_values = self._read_dem(dem_map)
@@ -42,12 +46,12 @@ def __init__(self, ctx, grid_id, geo_file_):
         log_message = f"""
         Correction
         Reading dem: {dem_map}
-        geopotential: {geo_file_}
+        geopotential: {geo_file}
         formula: {self._formula.replace('gem', self._gem_formula)}
         """
         self._log(log_message, 'INFO')
 
-        self._gem_missing_value, self._gem_values = self._read_geo(geo_file_, ctx)
+        self._gem_missing_value, self._gem_values = self._read_geo(geo_file, ctx)
 
     def correct(self, values):
         with np.errstate(over='ignore'):

src/pyg2p/main/readers/grib.py

@@ -15,6 +15,12 @@
 
 class GRIBReader(Loggable):
 
+    def __repr__(self):
+        return f'GRIBReader<{self._grib_file}>'
+
+    def __str__(self):
+        return self.__repr__()
+
     def __init__(self, grib_file, w_perturb=False):
         # codes_no_fail_on_wrong_length(True)
         super().__init__()

tests/__init__.py

@@ -58,6 +58,7 @@ def __init__(self, d, gribinterp=False):
         self.is_with_grib_interpolation = gribinterp
         self.configuration = self.conf
         self.configuration.intertables.data_path = os.path.abspath('tests/data/')
+        self.configuration.geopotentials.data_path = os.path.abspath('tests/data/')
 
     def get(self, param, default=None):
         return self._vars.get(param, default)
@@ -67,3 +68,4 @@ def geo_file(self, _):
 
     def create_select_cmd_for_aggregation_attrs(self):
         return {'shortName': '2t'}
+

tests/test_api.py

@@ -141,3 +141,85 @@ def test_cosmo_t24(self):
             assert np.allclose(diff, np.zeros(diff.shape), rtol=1.e-2, atol=1.e-3, equal_nan=True)
             i = int(i)
             i += 4
+
+    def test_iconeut24(self):
+        config = deepcopy(self.config)
+        config['inputFile'] = self.input_path.joinpath('dwd/L.grb')
+        config['start'] = 1440
+        config['end'] = 7200
+        config['ext'] = 4
+        config['perturbationNumber'] = None
+        config['Aggregation']['type'] = 'average'
+        config['Aggregation']['step'] = 1440
+        config['Parameter']['shortName'] = '2t'
+        config['Parameter']['applyConversion'] = 'k2c'
+        config['Parameter']['demMap'] = self.dem_map
+        config['Parameter']['gem'] = '(z/9.81)*0.0065'
+        config['Parameter']['correctionFormula'] = 'p+gem-dem*0.0065'
+        ctx = ApiContext(config)
+        api = Pyg2pApi(ctx)
+        out_values = api.execute()
+        assert len(out_values) == 5
+
+        i = 1
+        for step, val in out_values.items():
+            i = str(i).zfill(3)
+            reference = PCRasterReader(self.options['reference'].joinpath(f'dwd/T24a0000.{i}')).values
+            diff = np.abs(reference - val)
+            assert np.allclose(diff, np.zeros(diff.shape), rtol=1.e-2, atol=1.e-3, equal_nan=True)
+            i = int(i)
+            i += 4
+
+    def test_dwdt24(self):
+        config = deepcopy(self.config)
+        config['inputFile'] = self.input_path.joinpath('dwd/G.grb')
+        config['start'] = 8640
+        config['end'] = 10080
+        config['ext'] = 4
+        config['fmap'] = 21
+        config['perturbationNumber'] = None
+        config['Aggregation']['type'] = 'average'
+        config['Aggregation']['step'] = 1440
+        config['Parameter']['shortName'] = '2t'
+        config['Parameter']['applyConversion'] = 'k2c'
+        config['Parameter']['demMap'] = self.dem_map
+        config['Parameter']['gem'] = '(z/9.81)*0.0065'
+        config['Parameter']['correctionFormula'] = 'p+gem-dem*0.0065'
+        ctx = ApiContext(config)
+        api = Pyg2pApi(ctx)
+        out_values = api.execute()
+        assert len(out_values) == 2
+
+        i = 21
+        for step, val in out_values.items():
+            i = str(i).zfill(3)
+            reference = PCRasterReader(self.options['reference'].joinpath(f'dwd/T24a0000.{i}')).values
+            diff = np.abs(reference - val)
+            assert np.allclose(diff, np.zeros(diff.shape), rtol=1.e-2, atol=1.e-3, equal_nan=True)
+            i = int(i)
+            i += 4
+
+    @pytest.mark.slow
+    def test_createintertable(self):
+        config = deepcopy(self.config)
+        config['inputFile'] = 'tests/data/input.grib'
+        config['intertableDir'] = 'tests/data'
+        config['createIntertable'] = True
+        config['interpolationParallel'] = True
+        config['OutMaps']['Interpolation']['mode'] = 'invdist'
+        config['OutMaps']['Interpolation']['latMap'] = 'tests/data/lat.map'
+        config['OutMaps']['Interpolation']['lonMap'] = 'tests/data/lon.map'
+        config['OutMaps']['cloneMap'] = 'tests/data/dem.map'
+        config['perturbationNumber'] = None
+        config['Aggregation'] = None
+        config['Parameter']['shortName'] = '2t'
+        config['Parameter']['applyConversion'] = 'k2c'
+        config['Parameter']['demMap'] = 'tests/data/dem.map'
+        config['Parameter']['gem'] = '(z/9.81)*0.0065'
+        config['Parameter']['correctionFormula'] = 'p+gem-dem*0.0065'
+        ctx = ApiContext(config)
+        api = Pyg2pApi(ctx)
+        out_values = api.execute()
+        shape_target = PCRasterReader(config['OutMaps']['Interpolation']['latMap']).values.shape
+        assert shape_target == list(out_values.values())[0].shape
+        os.unlink('tests/data/tbl_pf10tp_550800_scipy_invdist.npy.gz')

tests/test_correction.py

@@ -20,14 +20,10 @@ def test_correction(self):
         demmap = PCRasterReader('tests/data/dem.map')
         dem = demmap.values
         dem_mv = demmap.mv
-        reader_geopotential = GRIBReader('tests/data/geopotential.grib')
-        geopotential = reader_geopotential.select_messages(shortName=GeopotentialsConfiguration.short_names)
-        z = geopotential.values_first_or_single_res[geopotential.first_step_range]
-        grid_id_geopotential = geopotential.grid_id
-        mv_geopotential = geopotential.missing_value
-        gem = np.where(z != mv_geopotential, (z / 9.81) * 0.0065, mv_geopotential)
+
         d = {
             'interpolation.dirs': {'user': os.path.abspath('tests/data/'), 'global': os.path.abspath('tests/data/')},
+            'geopotential.dirs': {'user': os.path.abspath('tests/data/'), 'global': os.path.abspath('tests/data/')},
             'interpolation.lonMap': 'tests/data/lon.map',
             'interpolation.latMap': 'tests/data/lat.map',
             'interpolation.mode': 'nearest',
@@ -37,15 +33,23 @@ def test_correction(self):
             'correction.gemFormula': '(z/9.81)*0.0065',
         }
         ctx = MockedExecutionContext(d, False)
-        corrector = Corrector.get_instance(ctx, grid_id)
+        corrector = Corrector(ctx, grid_id, geo_file='tests/data/geopotential.grib')
         values_in = messages.values_first_or_single_res[messages.first_step_range]
         lats, lons = messages.latlons
 
         interpolator = Interpolator(ctx, missing)
         values_resampled = interpolator.interpolate_scipy(lats, lons, values_in, grid_id, messages.grid_details)
-        gem_resampled = interpolator.interpolate_scipy(lats, lons, gem, grid_id_geopotential, geopotential.grid_details)
         values_out = corrector.correct(values_resampled)
+
+        reader_geopotential = GRIBReader('tests/data/geopotential.grib')
+        geopotential = reader_geopotential.select_messages(shortName=GeopotentialsConfiguration.short_names)
+        z = geopotential.values_first_or_single_res[geopotential.first_step_range]
+        grid_id_geopotential = geopotential.grid_id
+        mv_geopotential = geopotential.missing_value
+        gem = np.where(z != mv_geopotential, (z / 9.81) * 0.0065, mv_geopotential)
+        gem_resampled = interpolator.interpolate_scipy(lats, lons, gem, grid_id_geopotential, geopotential.grid_details)
         reference = np.where((dem != dem_mv) & (values_resampled != dem_mv) & (gem_resampled != dem_mv),
                              values_resampled + gem_resampled - dem * 0.0065,
                              dem_mv)
+
         assert np.allclose(values_out, reference)