diff --git a/satpy/readers/fci_l1c_nc.py b/satpy/readers/fci_l1c_nc.py index 7954d4866d..a03684fe89 100644 --- a/satpy/readers/fci_l1c_nc.py +++ b/satpy/readers/fci_l1c_nc.py @@ -158,6 +158,15 @@ "grid_width": 5568}, } +NONSHAREABLE_VARIABLE_ENDINGS = [ + "index", + "time", + "measured/effective_radiance", + "measured/y", + "position_row", + "index_map", + "pixel_quality"] + def _get_aux_data_name_from_dsname(dsname): aux_data_name = [key for key in AUX_DATA.keys() if key in dsname] @@ -700,6 +709,31 @@ def calibrate_rad_to_refl(self, radiance, key): res = 100 * radiance * np.float32(np.pi) * np.float32(sun_earth_distance) ** np.float32(2) / cesi return res + def _collect_listed_variables(self, file_handle, listed_variables, filetype_info): + listed_variables = self._collect_from_inter_segment_cache(listed_variables, filetype_info) + super()._collect_listed_variables(file_handle, listed_variables, filetype_info) + self._store_shared_info(filetype_info) + + def _collect_from_inter_segment_cache(self, listed_variables, filetype_info): + if "shared_info" in filetype_info: + shared_info = filetype_info["shared_info"] + for key in shared_info: + self.file_content[key] = shared_info[key] + try: + listed_variables.remove(key) + except ValueError: + pass + return listed_variables + + def _store_shared_info(self, filetype_info): + if "shared_info" not in filetype_info: + shared_info = {} + for key in self.file_content: + if any(key.endswith(k) for k in NONSHAREABLE_VARIABLE_ENDINGS): + continue + shared_info[key] = self.file_content[key] + filetype_info["shared_info"] = shared_info + def _ensure_dataarray(arr): if not isinstance(arr, xr.DataArray): diff --git a/satpy/readers/netcdf_utils.py b/satpy/readers/netcdf_utils.py index c8b8a3f85f..db4db9934a 100644 --- a/satpy/readers/netcdf_utils.py +++ b/satpy/readers/netcdf_utils.py @@ -111,7 +111,9 @@ def __init__(self, filename, filename_info, filetype_info, listed_variables = filetype_info.get("required_netcdf_variables") if listed_variables: - self._collect_listed_variables(file_handle, listed_variables) + variable_name_replacements = self.filetype_info.get("variable_name_replacements") + listed_variables = self._get_required_variable_names(listed_variables, variable_name_replacements) + self._collect_listed_variables(file_handle, listed_variables, filetype_info) else: self.collect_metadata("", file_handle) self.collect_dimensions("", file_handle) @@ -168,9 +170,8 @@ def _collect_variable_info(self, var_name, var_obj): self.file_content[var_name + "/dimensions"] = var_obj.dimensions self._collect_attrs(var_name, var_obj) - def _collect_listed_variables(self, file_handle, listed_variables): - variable_name_replacements = self.filetype_info.get("variable_name_replacements") - for itm in self._get_required_variable_names(listed_variables, variable_name_replacements): + def _collect_listed_variables(self, file_handle, listed_variables, filetype_info): + for itm in listed_variables: parts = itm.split("/") grp = file_handle for p in parts[:-1]: diff --git a/satpy/tests/reader_tests/test_fci_l1c_nc.py b/satpy/tests/reader_tests/test_fci_l1c_nc.py index 04f2d48930..8e24212b67 100644 --- a/satpy/tests/reader_tests/test_fci_l1c_nc.py +++ b/satpy/tests/reader_tests/test_fci_l1c_nc.py @@ -921,3 +921,81 @@ def test_bad_xy_coords(self, reader_configs): np.array([-5568000.227139, -5368000.221262, 5568000.100073, -5568000.227139]), decimal=2) + + +def _mock_np2str(val): + return val + + +def parse_fdhsi_filename_info(filename): + """Parse filename info from the filename.""" + from trollsift import parse + + fmt = ("{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-FDHSI-" + "{coverage}-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_" + "{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_" + "{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc") + filename_info = parse(fmt, filename) + return filename_info + + +def test_reusing_shared_segment_info(): + """Test that certain data are reused after the first segment has been handled. + + This is the case where something has been already handled and stored to filetype_info["shared_info"] dict. + """ + filename = TEST_FILENAMES["fdhsi"][0] + filename_info = parse_fdhsi_filename_info(filename) + filetype_info = { + "shared_info": { + "attr/platform": "mock-satellite" + }, + "required_netcdf_variables": ["attr/platform"] + } + + with mock.patch("satpy.readers.netcdf_utils.NetCDF4FsspecFileHandler._get_file_handle"): + with mock.patch("satpy.readers.netcdf_utils.np2str", _mock_np2str): + fh_ = FCIL1cNCFileHandler(filename, filename_info, filetype_info) + + assert fh_.file_content["attr/platform"] == filetype_info["shared_info"]["attr/platform"] + + +def test_store_shared_segment_info(): + """Test that certain data are collected to filetype_info["shared_info"] from the first segment.""" + filename = TEST_FILENAMES["fdhsi"][0] + filename_info = parse_fdhsi_filename_info(filename) + filetype_info = {"required_netcdf_variables": ["attr/platform"]} + + with mock.patch("satpy.readers.netcdf_utils.NetCDF4FsspecFileHandler._get_file_handle") as get_file_handle: + get_file_handle.return_value.platform = "mock-satellite" + with mock.patch("satpy.readers.netcdf_utils.np2str", _mock_np2str): + _ = FCIL1cNCFileHandler(filename, filename_info, filetype_info) + + assert filetype_info["shared_info"]["attr/platform"] == "mock-satellite" + + +def test_store_shared_segment_info_ignore_nonshareable(): + """Test that nonshareable segment info are not collected for sharing.""" + variables = [ + "data/ir105/measured/effective_radiance", + "data/ir105/measured/y", + "data/ir105/measured/end_position_row", + "data/ir105/measured/start_position_row", + "data/ir105/measured/index_map", + "data/ir105/measured/pixel_quality", + "index", + "time", + ] + + filename = TEST_FILENAMES["fdhsi"][0] + filename_info = parse_fdhsi_filename_info(filename) + filetype_info = {"required_netcdf_variables": variables} + + with mock.patch("satpy.readers.netcdf_utils.NetCDF4FsspecFileHandler._get_file_handle") as get_file_handle: + for itm in variables: + setattr(get_file_handle.return_value, itm, "mock-value") + with mock.patch("satpy.readers.netcdf_utils.np2str", _mock_np2str): + _ = FCIL1cNCFileHandler(filename, filename_info, filetype_info) + + for itm in variables: + assert itm not in filetype_info["shared_info"]