Create decumulation test

.gitignore

@@ -53,6 +53,7 @@ tests/**/*aux.xml
 !src/lisfloodutilities/gridding/**/*.nc
 tests/data/gridding/meteo_out/**/*.nc
 tests/data/gridding/meteo_out/**/*.tiff
+tests/data/gridding/meteo_out/**/*.kiwis
 .settings
 .project
 .pydevproject

src/lisfloodutilities/gridding/configuration/1arcmin/config_pr.txt

@@ -22,7 +22,7 @@ LONG_NAME = Daily Accumulated Precipitation
 KIWIS_FILTER_PLUGIN_CLASSES = {'ObservationsKiwisFilter': {'1303': 100.0, '1329': 100.0}}
 
 # 1310/HNMS until 31/12/2000, 1295/MARS until 01/01/2008, 1304/EURO4M-APGD until 31/12/2008, 1302/CarpatClim until 31/12/2010 and 1280/IMGW until 31/12/2020
-KIWIS_FILTER_DECUMULATION_CONFIG = [{'START': '1989-12-31 00:00:00', 'END': '2000-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1295': 1.5, '1302': 1.5, '1304': 1.5, '1310': 1.5}},{'START': '2001-01-01 00:00:00', 'END': '2007-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1295': 1.5, '1302': 1.5, '1304': 1.5}},{'START': '2008-01-01 00:00:00', 'END': '2008-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1302': 1.5, '1304': 1.5}},{'START': '2009-01-01 00:00:00', 'END': '2010-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1302': 1.5},{'START': '2011-01-01 00:00:00', 'END': '2020-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5}}]
+KIWIS_FILTER_DECUMULATION_CONFIG = [{'START': '1989-12-31 00:00:00', 'END': '2000-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1295': 1.5, '1302': 1.5, '1304': 1.5, '1310': 1.5}},{'START': '2001-01-01 00:00:00', 'END': '2007-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1295': 1.5, '1302': 1.5, '1304': 1.5}},{'START': '2008-01-01 00:00:00', 'END': '2008-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1302': 1.5, '1304': 1.5}},{'START': '2009-01-01 00:00:00', 'END': '2010-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1302': 1.5}},{'START': '2011-01-01 00:00:00', 'END': '2020-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5}}]
 
 
 [VAR_TIME]

src/lisfloodutilities/gridding/decumulate_daily_grids.py

@@ -373,6 +373,7 @@ def main(argv):
 
             run(conf_24h, conf_6h, kiwis_24h_06am_path, kiwis_6h_12pm_path, kiwis_6h_18pm_path,
                 kiwis_6h_12am_path, kiwis_6h_06am_path, input_path_6h=kiwis_6h_folder_path, output_path=output_path)
+        return 0
     except Exception as e:
         indent = len(program_name) * " "
         sys.stderr.write(program_name + ": " + repr(e) + "\n")

src/lisfloodutilities/gridding/lib/filters.py

@@ -91,9 +91,9 @@ def print_statistics(self, df: pd.DataFrame):
                                                        column_quality_code=self.COL_QUALITY_CODE,
                                                        quality_code=self.QUALITY_CODE_WRONG)
         new_df.drop(columns=[self.COL_QUALITY_CODE, 'count'], inplace=True)
-        new_df = new_df.groupby(self.COL_PROVIDER_ID)[self.QUALITY_CODE_VALID,
-                                                      self.QUALITY_CODE_SUSPICIOUS,
-                                                      self.QUALITY_CODE_WRONG].sum()
+        new_df = new_df.groupby(self.COL_PROVIDER_ID)[[self.QUALITY_CODE_VALID,
+                                                       self.QUALITY_CODE_SUSPICIOUS,
+                                                       self.QUALITY_CODE_WRONG]].sum()
         new_df.reset_index(inplace=True)
         for index, row in new_df.iterrows():
             provider_id = row[self.COL_PROVIDER_ID]

tests/test_decumulate.py

@@ -0,0 +1,44 @@
+import os
+from pathlib import Path
+from lisfloodutilities.gridding.decumulate_daily_grids import main
+
+
+cur_folder = os.path.dirname(os.path.realpath(__file__))
+
+
+class TestDecumulate:
+
+    def check_file_content(self, file1: Path, file2: Path):
+        with open(file1, 'r') as f1, open(file2, 'r') as f2:
+            content1 = f1.read()
+            content2 = f2.read()
+        file_name1 = str(file1).replace(self.base_data_folder, '')
+        file_name2 = str(file2).replace(self.base_data_folder, '')
+        assert content1 == content2, f'Files [{file_name1}] and [{file_name2}] do not have the same content.'
+
+    def test_decumulate_kiwis(self):
+        configuration_base_folder = Path(os.path.join(cur_folder, '../src/lisfloodutilities/gridding/configuration'))
+        self.base_data_folder = os.path.join(cur_folder, 'data/decumulate/')
+        input_folder = Path(os.path.join(self.base_data_folder, 'meteo_in'))
+        reference_folder = Path(os.path.join(self.base_data_folder, 'reference/pr6'))
+        output_folder = Path(os.path.join(self.base_data_folder, 'meteo_out/pr6'))
+
+        output_folder.mkdir(parents=True, exist_ok=True)
+        # Clean the output folder
+        for filename_kiwis in output_folder.rglob('*.kiwis'):
+            os.remove(filename_kiwis)
+
+        argv = ['--conf', '1arcmin', '--pathconf', f'{configuration_base_folder}',
+                '--var24h', 'pr', '--var6h', 'pr6', '--out', f'{output_folder}',
+                '--pr24h', f'{input_folder}/pr/', '-s', '20051226000000', '-e', '20051226060001',
+                '--pr6h', f'{input_folder}/pr6/']
+
+        retval = main(argv)
+
+        assert retval == 0, f'Got return value {retval}. Some error occurred while processing.'
+
+        for filename_kiwis in reference_folder.rglob('*.kiwis'):
+            out_file = str(filename_kiwis).replace(str(reference_folder), str(output_folder))
+            assert os.path.exists(out_file), f'File [{out_file}] was not generated.'
+            self.check_file_content(filename_kiwis, Path(out_file))
+