diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index 0e6670533f..74889e5066 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -142,7 +142,9 @@ def callback(cube, field, filename): class Future(threading.local): """Run-time configuration controller.""" - def __init__(self, datum_support=False, pandas_ndim=False): + def __init__( + self, datum_support=False, pandas_ndim=False, save_split_attrs=False + ): """ A container for run-time options controls. @@ -164,6 +166,11 @@ def __init__(self, datum_support=False, pandas_ndim=False): pandas_ndim : bool, default=False See :func:`iris.pandas.as_data_frame` for details - opts in to the newer n-dimensional behaviour. + save_split_attrs : bool, default=False + Save "global" and "local" cube attributes to netcdf in appropriately + different ways : "global" ones are saved as dataset attributes, where + possible, while "local" ones are saved as data-variable attributes. + See :func:`iris.fileformats.netcdf.saver.save`. """ # The flag 'example_future_flag' is provided as a reference for the @@ -175,12 +182,15 @@ def __init__(self, datum_support=False, pandas_ndim=False): # self.__dict__['example_future_flag'] = example_future_flag self.__dict__["datum_support"] = datum_support self.__dict__["pandas_ndim"] = pandas_ndim + self.__dict__["save_split_attrs"] = save_split_attrs def __repr__(self): # msg = ('Future(example_future_flag={})') # return msg.format(self.example_future_flag) - msg = "Future(datum_support={}, pandas_ndim={})" - return msg.format(self.datum_support, self.pandas_ndim) + msg = "Future(datum_support={}, pandas_ndim={}, save_split_attrs={})" + return msg.format( + self.datum_support, self.pandas_ndim, self.save_split_attrs + ) # deprecated_options = {'example_future_flag': 'warning',} deprecated_options = {} diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 8bb9d7c00e..0d1f531bf9 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -930,19 +930,19 @@ def _normalise_attrs( return attributes @property - def locals(self): + def locals(self) -> LimitedAttributeDict: return self._locals @locals.setter - def locals(self, attributes): + def locals(self, attributes: Optional[Mapping]): self._locals = self._normalise_attrs(attributes) @property - def globals(self): + def globals(self) -> LimitedAttributeDict: return self._globals @globals.setter - def globals(self, attributes): + def globals(self, attributes: Optional[Mapping]): self._globals = self._normalise_attrs(attributes) # @@ -1335,8 +1335,12 @@ def _names(self): # # Ensure that .attributes is always a :class:`CubeAttrsDict`. # - @CFVariableMixin.attributes.setter - def attributes(self, attributes): + @property + def attributes(self) -> CubeAttrsDict: + return super().attributes + + @attributes.setter + def attributes(self, attributes: Optional[Mapping]): """ An override to CfVariableMixin.attributes.setter, which ensures that Cube attributes are stored in a way which distinguishes global + local ones. diff --git a/lib/iris/fileformats/netcdf/saver.py b/lib/iris/fileformats/netcdf/saver.py index 312eea9c43..552eadb070 100644 --- a/lib/iris/fileformats/netcdf/saver.py +++ b/lib/iris/fileformats/netcdf/saver.py @@ -541,6 +541,10 @@ def write( matching keys will become attributes on the data variable rather than global attributes. + .. Note:: + + Has no effect if :attr:`iris.FUTURE.save_split_attrs` is ``True``. + * unlimited_dimensions (iterable of strings and/or :class:`iris.coords.Coord` objects): List of coordinate names (or coordinate objects) @@ -633,6 +637,9 @@ def write( 3 files that do not use HDF5. """ + # TODO: when iris.FUTURE.save_split_attrs defaults to True, we can deprecate the + # "local_keys" arg, and finally remove it when we finally remove the + # save_split_attrs switch. if unlimited_dimensions is None: unlimited_dimensions = [] @@ -709,20 +716,23 @@ def write( # aux factory in the cube. self._add_aux_factories(cube, cf_var_cube, cube_dimensions) - # Add data variable-only attribute names to local_keys. - if local_keys is None: - local_keys = set() - else: - local_keys = set(local_keys) - local_keys.update(_CF_DATA_ATTRS, _UKMO_DATA_ATTRS) - - # Add global attributes taking into account local_keys. - global_attributes = { - k: v - for k, v in cube.attributes.items() - if (k not in local_keys and k.lower() != "conventions") - } - self.update_global_attributes(global_attributes) + if not iris.FUTURE.save_split_attrs: + # In the "old" way, we update global attributes as we go. + # Add data variable-only attribute names to local_keys. + if local_keys is None: + local_keys = set() + else: + local_keys = set(local_keys) + local_keys.update(_CF_DATA_ATTRS, _UKMO_DATA_ATTRS) + + # Add global attributes taking into account local_keys. + cube_attributes = cube.attributes + global_attributes = { + k: v + for k, v in cube_attributes.items() + if (k not in local_keys and k.lower() != "conventions") + } + self.update_global_attributes(global_attributes) if cf_profile_available: cf_patch = iris.site_configuration.get("cf_patch") @@ -778,6 +788,9 @@ def update_global_attributes(self, attributes=None, **kwargs): CF global attributes to be updated. """ + # TODO: when when iris.FUTURE.save_split_attrs is removed, this routine will + # only be called once: it can reasonably be renamed "_set_global_attributes", + # and the 'kwargs' argument can be removed. if attributes is not None: # Handle sequence e.g. [('fruit', 'apple'), ...]. if not hasattr(attributes, "keys"): @@ -2195,6 +2208,8 @@ def _create_cf_data_variable( """ Create CF-netCDF data variable for the cube and any associated grid mapping. + # TODO: when iris.FUTURE.save_split_attrs is removed, the 'local_keys' arg can + # be removed. Args: @@ -2219,6 +2234,8 @@ def _create_cf_data_variable( The newly created CF-netCDF data variable. """ + # TODO: when iris.FUTURE.save_split_attrs is removed, the 'local_keys' arg can + # be removed. # Get the values in a form which is valid for the file format. data = self._ensure_valid_dtype(cube.core_data(), "cube", cube) @@ -2307,16 +2324,20 @@ def set_packing_ncattrs(cfvar): if cube.units.calendar: _setncattr(cf_var, "calendar", cube.units.calendar) - # Add data variable-only attribute names to local_keys. - if local_keys is None: - local_keys = set() + if iris.FUTURE.save_split_attrs: + attr_names = cube.attributes.locals.keys() else: - local_keys = set(local_keys) - local_keys.update(_CF_DATA_ATTRS, _UKMO_DATA_ATTRS) + # Add data variable-only attribute names to local_keys. + if local_keys is None: + local_keys = set() + else: + local_keys = set(local_keys) + local_keys.update(_CF_DATA_ATTRS, _UKMO_DATA_ATTRS) + + # Add any cube attributes whose keys are in local_keys as + # CF-netCDF data variable attributes. + attr_names = set(cube.attributes).intersection(local_keys) - # Add any cube attributes whose keys are in local_keys as - # CF-netCDF data variable attributes. - attr_names = set(cube.attributes).intersection(local_keys) for attr_name in sorted(attr_names): # Do not output 'conventions' attribute. if attr_name.lower() == "conventions": @@ -2600,9 +2621,15 @@ def save( Save cube(s) to a netCDF file, given the cube and the filename. * Iris will write CF 1.7 compliant NetCDF files. - * The attributes dictionaries on each cube in the saved cube list - will be compared and common attributes saved as NetCDF global - attributes where appropriate. + * **If split-attribute saving is disabled**, i.e. + :data:`iris.FUTURE`\\ ``.save_split_attrs`` is ``False``, then attributes + dictionaries on each cube in the saved cube list will be compared, and common + attributes saved as NetCDF global attributes where appropriate. + + Or, **when split-attribute saving is enabled**, then ``cube.attributes.locals`` + are always saved as attributes of data-variables, and ``cube.attributes.globals`` + are saved as global (dataset) attributes, where possible. + Since the 2 types are now distinguished : see :class:`~iris.cube.CubeAttrsDict`. * Keyword arguments specifying how to save the data are applied to each cube. To use different settings for different cubes, use the NetCDF Context manager (:class:`~Saver`) directly. @@ -2635,6 +2662,8 @@ def save( An interable of cube attribute keys. Any cube attributes with matching keys will become attributes on the data variable rather than global attributes. + **NOTE:** this is *ignored* if 'split-attribute saving' is **enabled**, + i.e. when ``iris.FUTURE.save_split_attrs`` is ``True``. * unlimited_dimensions (iterable of strings and/or :class:`iris.coords.Coord` objects): @@ -2773,26 +2802,114 @@ def save( else: cubes = cube - if local_keys is None: + # Decide which cube attributes will be saved as "global" attributes + # NOTE: in 'legacy' mode, when iris.FUTURE.save_split_attrs == False, this code + # section derives a common value for 'local_keys', which is passed to 'Saver.write' + # when saving each input cube. The global attributes are then created by a call + # to "Saver.update_global_attributes" within each 'Saver.write' call (which is + # obviously a bit redundant!), plus an extra one to add 'Conventions'. + # HOWEVER, in `split_attrs` mode (iris.FUTURE.save_split_attrs == False), this code + # instead constructs a 'global_attributes' dictionary, and outputs that just once, + # after writing all the input cubes. + if iris.FUTURE.save_split_attrs: + # We don't actually use 'local_keys' in this case. + # TODO: can remove this when the iris.FUTURE.save_split_attrs is removed. local_keys = set() + + # Find any collisions in the cube global attributes and "demote" all those to + # local attributes (where possible, else warn they are lost). + # N.B. "collision" includes when not all cubes *have* that attribute. + global_names = set() + for cube in cubes: + global_names |= set(cube.attributes.globals.keys()) + + # Fnd any global attributes which are not the same on *all* cubes. + def attr_values_equal(val1, val2): + # An equality test which also works when some values are numpy arrays (!) + # As done in :meth:`iris.common.mixin.LimitedAttributeDict.__eq__`. + match = val1 == val2 + try: + match = bool(match) + except ValueError: + match = match.all() + return match + + cube0 = cubes[0] + invalid_globals = set( + [ + attrname + for attrname in global_names + if not all( + attr_values_equal( + cube.attributes.globals.get(attrname), + cube0.attributes.globals.get(attrname), + ) + for cube in cubes[1:] + ) + ] + ) + + # Establish all the global attributes which we will write to the file (at end). + global_attributes = { + attr: cube0.attributes.globals.get(attr) + for attr in global_names - invalid_globals + } + if invalid_globals: + # Some cubes have different global attributes: modify cubes as required. + warnings.warn( + f"Saving the cube global attributes {sorted(invalid_globals)} as local " + "(i.e. data-variable) attributes, where possible, since they are not " + "the same on all input cubes." + ) + cubes = cubes.copy() # avoiding modifying the actual input arg. + for i_cube in range(len(cubes)): + # We iterate over cube *index*, so we can replace the list entries with + # with cube *copies* -- just to avoid changing our call args. + cube = cubes[i_cube] + demote_attrs = set(cube.attributes.globals) & invalid_globals + if any(demote_attrs): + # Catch any demoted attrs where there is already a local version + blocked_attrs = demote_attrs & set(cube.attributes.locals) + if blocked_attrs: + warnings.warn( + f"Global cube attributes {sorted(blocked_attrs)} " + f'of cube "{cube.name()}" were not saved, overlaid ' + "by existing local attributes with the same names." + ) + demote_attrs -= blocked_attrs + if demote_attrs: + # This cube contains some 'demoted' global attributes. + # Replace input cube with a copy, so we can modify attributes. + cube = cube.copy() + cubes[i_cube] = cube + for attr in demote_attrs: + # move global to local + value = cube.attributes.globals.pop(attr) + cube.attributes.locals[attr] = value + else: - local_keys = set(local_keys) - - # Determine the attribute keys that are common across all cubes and - # thereby extend the collection of local_keys for attributes - # that should be attributes on data variables. - attributes = cubes[0].attributes - common_keys = set(attributes) - for cube in cubes[1:]: - keys = set(cube.attributes) - local_keys.update(keys.symmetric_difference(common_keys)) - common_keys.intersection_update(keys) - different_value_keys = [] - for key in common_keys: - if np.any(attributes[key] != cube.attributes[key]): - different_value_keys.append(key) - common_keys.difference_update(different_value_keys) - local_keys.update(different_value_keys) + # Legacy mode: calculate "local_keys" to control which attributes are local + # and which global. + if local_keys is None: + local_keys = set() + else: + local_keys = set(local_keys) + + # Determine the attribute keys that are common across all cubes and + # thereby extend the collection of local_keys for attributes + # that should be attributes on data variables. + attributes = cubes[0].attributes + common_keys = set(attributes) + for cube in cubes[1:]: + keys = set(cube.attributes) + local_keys.update(keys.symmetric_difference(common_keys)) + common_keys.intersection_update(keys) + different_value_keys = [] + for key in common_keys: + if np.any(attributes[key] != cube.attributes[key]): + different_value_keys.append(key) + common_keys.difference_update(different_value_keys) + local_keys.update(different_value_keys) def is_valid_packspec(p): """Only checks that the datatype is valid.""" @@ -2894,7 +3011,12 @@ def is_valid_packspec(p): warnings.warn(msg) # Add conventions attribute. - sman.update_global_attributes(Conventions=conventions) + if iris.FUTURE.save_split_attrs: + # In the "new way", we just create all the global attributes at once. + global_attributes["Conventions"] = conventions + sman.update_global_attributes(global_attributes) + else: + sman.update_global_attributes(Conventions=conventions) if compute: # No more to do, since we used Saver(compute=True). diff --git a/lib/iris/tests/integration/attrs_matrix_results_load.json b/lib/iris/tests/integration/attrs_matrix_results_load.json new file mode 100644 index 0000000000..a1d37708a9 --- /dev/null +++ b/lib/iris/tests/integration/attrs_matrix_results_load.json @@ -0,0 +1,1019 @@ +{ + "case_singlevar_localonly": { + "input": "G-La", + "localstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "G-La" + ] + }, + "globalstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "G-La" + ] + }, + "userstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "G-La" + ] + } + }, + "case_singlevar_globalonly": { + "input": "GaL-", + "localstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "GaL-" + ] + }, + "globalstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "GaL-" + ] + }, + "userstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "GaL-" + ] + } + }, + "case_singlevar_glsame": { + "input": "GaLa", + "localstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "GaLa" + ] + }, + "globalstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "GaLa" + ] + }, + "userstyle": { + "legacy": [ + "G-La" + ], + "newstyle": [ + "GaLa" + ] + } + }, + "case_singlevar_gldiffer": { + "input": "GaLb", + "localstyle": { + "legacy": [ + "G-Lb" + ], + "newstyle": [ + "GaLb" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lb" + ], + "newstyle": [ + "GaLb" + ] + }, + "userstyle": { + "legacy": [ + "G-Lb" + ], + "newstyle": [ + "GaLb" + ] + } + }, + "case_multivar_same_noglobal": { + "input": "G-Laa", + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-Laa" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-Laa" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-Laa" + ] + } + }, + "case_multivar_same_sameglobal": { + "input": "GaLaa", + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLaa" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLaa" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLaa" + ] + } + }, + "case_multivar_same_diffglobal": { + "input": "GaLbb", + "localstyle": { + "legacy": [ + "G-Lbb" + ], + "newstyle": [ + "GaLbb" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lbb" + ], + "newstyle": [ + "GaLbb" + ] + }, + "userstyle": { + "legacy": [ + "G-Lbb" + ], + "newstyle": [ + "GaLbb" + ] + } + }, + "case_multivar_differ_noglobal": { + "input": "G-Lab", + "localstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lab" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lab" + ] + }, + "userstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lab" + ] + } + }, + "case_multivar_differ_diffglobal": { + "input": "GaLbc", + "localstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLbc" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLbc" + ] + }, + "userstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLbc" + ] + } + }, + "case_multivar_differ_sameglobal": { + "input": "GaLab", + "localstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLab" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLab" + ] + }, + "userstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLab" + ] + } + }, + "case_multivar_1none_noglobal": { + "input": "G-La-", + "localstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-La-" + ] + }, + "globalstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-La-" + ] + }, + "userstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-La-" + ] + } + }, + "case_multivar_1none_diffglobal": { + "input": "GaLb-", + "localstyle": { + "legacy": [ + "G-Lba" + ], + "newstyle": [ + "GaLb-" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lba" + ], + "newstyle": [ + "GaLb-" + ] + }, + "userstyle": { + "legacy": [ + "G-Lba" + ], + "newstyle": [ + "GaLb-" + ] + } + }, + "case_multivar_1none_sameglobal": { + "input": "GaLa-", + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLa-" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLa-" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLa-" + ] + } + }, + "case_multisource_gsame_lnone": { + "input": [ + "GaL-", + "GaL-" + ], + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaL--" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaL--" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaL--" + ] + } + }, + "case_multisource_gsame_lallsame": { + "input": [ + "GaLa", + "GaLa" + ], + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLaa" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLaa" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLaa" + ] + } + }, + "case_multisource_gsame_l1same1none": { + "input": [ + "GaLa", + "GaL-" + ], + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLa-" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLa-" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "GaLa-" + ] + } + }, + "case_multisource_gsame_l1same1other": { + "input": [ + "GaLa", + "GaLb" + ], + "localstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLab" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLab" + ] + }, + "userstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLab" + ] + } + }, + "case_multisource_gsame_lallother": { + "input": [ + "GaLb", + "GaLb" + ], + "localstyle": { + "legacy": [ + "G-Lbb" + ], + "newstyle": [ + "GaLbb" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lbb" + ], + "newstyle": [ + "GaLbb" + ] + }, + "userstyle": { + "legacy": [ + "G-Lbb" + ], + "newstyle": [ + "GaLbb" + ] + } + }, + "case_multisource_gsame_lalldiffer": { + "input": [ + "GaLb", + "GaLc" + ], + "localstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLbc" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLbc" + ] + }, + "userstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLbc" + ] + } + }, + "case_multisource_gnone_l1one1none": { + "input": [ + "G-La", + "G-L-" + ], + "localstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-La-" + ] + }, + "globalstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-La-" + ] + }, + "userstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-La-" + ] + } + }, + "case_multisource_gnone_l1one1same": { + "input": [ + "G-La", + "G-La" + ], + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-Laa" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-Laa" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-Laa" + ] + } + }, + "case_multisource_gnone_l1one1other": { + "input": [ + "G-La", + "G-Lb" + ], + "localstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lab" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lab" + ] + }, + "userstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lab" + ] + } + }, + "case_multisource_g1none_lnone": { + "input": [ + "GaL-", + "G-L-" + ], + "localstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-L-", + "GaL-" + ] + }, + "globalstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-L-", + "GaL-" + ] + }, + "userstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-L-", + "GaL-" + ] + } + }, + "case_multisource_g1none_l1same1none": { + "input": [ + "GaLa", + "G-L-" + ], + "localstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-L-", + "GaLa" + ] + }, + "globalstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-L-", + "GaLa" + ] + }, + "userstyle": { + "legacy": [ + "G-La-" + ], + "newstyle": [ + "G-L-", + "GaLa" + ] + } + }, + "case_multisource_g1none_l1none1same": { + "input": [ + "GaL-", + "G-La" + ], + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-La", + "GaL-" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-La", + "GaL-" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-La", + "GaL-" + ] + } + }, + "case_multisource_g1none_l1diff1none": { + "input": [ + "GaLb", + "G-L-" + ], + "localstyle": { + "legacy": [ + "G-Lb-" + ], + "newstyle": [ + "G-L-", + "GaLb" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lb-" + ], + "newstyle": [ + "G-L-", + "GaLb" + ] + }, + "userstyle": { + "legacy": [ + "G-Lb-" + ], + "newstyle": [ + "G-L-", + "GaLb" + ] + } + }, + "case_multisource_g1none_l1none1diff": { + "input": [ + "GaL-", + "G-Lb" + ], + "localstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lb", + "GaL-" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lb", + "GaL-" + ] + }, + "userstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "G-Lb", + "GaL-" + ] + } + }, + "case_multisource_g1none_lallsame": { + "input": [ + "GaLa", + "G-La" + ], + "localstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-La", + "GaLa" + ] + }, + "globalstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-La", + "GaLa" + ] + }, + "userstyle": { + "legacy": [ + "G-Laa" + ], + "newstyle": [ + "G-La", + "GaLa" + ] + } + }, + "case_multisource_g1none_lallother": { + "input": [ + "GaLc", + "G-Lc" + ], + "localstyle": { + "legacy": [ + "G-Lcc" + ], + "newstyle": [ + "G-Lc", + "GaLc" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lcc" + ], + "newstyle": [ + "G-Lc", + "GaLc" + ] + }, + "userstyle": { + "legacy": [ + "G-Lcc" + ], + "newstyle": [ + "G-Lc", + "GaLc" + ] + } + }, + "case_multisource_gdiff_lnone": { + "input": [ + "GaL-", + "GbL-" + ], + "localstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaL-", + "GbL-" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaL-", + "GbL-" + ] + }, + "userstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaL-", + "GbL-" + ] + } + }, + "case_multisource_gdiff_l1same1none": { + "input": [ + "GaLa", + "GbL-" + ], + "localstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLa", + "GbL-" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLa", + "GbL-" + ] + }, + "userstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLa", + "GbL-" + ] + } + }, + "case_multisource_gdiff_l1diff1none": { + "input": [ + "GaLb", + "GcL-" + ], + "localstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLb", + "GcL-" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLb", + "GcL-" + ] + }, + "userstyle": { + "legacy": [ + "G-Lbc" + ], + "newstyle": [ + "GaLb", + "GcL-" + ] + } + }, + "case_multisource_gdiff_lallsame": { + "input": [ + "GaLa", + "GbLb" + ], + "localstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLa", + "GbLb" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLa", + "GbLb" + ] + }, + "userstyle": { + "legacy": [ + "G-Lab" + ], + "newstyle": [ + "GaLa", + "GbLb" + ] + } + }, + "case_multisource_gdiff_lallother": { + "input": [ + "GaLc", + "GbLc" + ], + "localstyle": { + "legacy": [ + "G-Lcc" + ], + "newstyle": [ + "GaLc", + "GbLc" + ] + }, + "globalstyle": { + "legacy": [ + "G-Lcc" + ], + "newstyle": [ + "GaLc", + "GbLc" + ] + }, + "userstyle": { + "legacy": [ + "G-Lcc" + ], + "newstyle": [ + "GaLc", + "GbLc" + ] + } + } +} \ No newline at end of file diff --git a/lib/iris/tests/integration/attrs_matrix_results_roundtrip.json b/lib/iris/tests/integration/attrs_matrix_results_roundtrip.json new file mode 100644 index 0000000000..3446c7f312 --- /dev/null +++ b/lib/iris/tests/integration/attrs_matrix_results_roundtrip.json @@ -0,0 +1,983 @@ +{ + "case_singlevar_localonly": { + "input": "G-La", + "localstyle": { + "unsplit": [ + "G-La" + ], + "split": [ + "G-La" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "G-La" + ] + }, + "userstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "G-La" + ] + } + }, + "case_singlevar_globalonly": { + "input": "GaL-", + "localstyle": { + "unsplit": [ + "G-La" + ], + "split": [ + "GaL-" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "GaL-" + ] + }, + "userstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "GaL-" + ] + } + }, + "case_singlevar_glsame": { + "input": "GaLa", + "localstyle": { + "unsplit": [ + "G-La" + ], + "split": [ + "GaLa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "GaLa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "GaLa" + ] + } + }, + "case_singlevar_gldiffer": { + "input": "GaLb", + "localstyle": { + "unsplit": [ + "G-Lb" + ], + "split": [ + "GaLb" + ] + }, + "globalstyle": { + "unsplit": [ + "GbL-" + ], + "split": [ + "GaLb" + ] + }, + "userstyle": { + "unsplit": [ + "GbL-" + ], + "split": [ + "GaLb" + ] + } + }, + "case_multivar_same_noglobal": { + "input": "G-Laa", + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "G-Laa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + } + }, + "case_multivar_same_sameglobal": { + "input": "GaLaa", + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaLaa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLaa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLaa" + ] + } + }, + "case_multivar_same_diffglobal": { + "input": "GaLbb", + "localstyle": { + "unsplit": [ + "G-Lbb" + ], + "split": [ + "GaLbb" + ] + }, + "globalstyle": { + "unsplit": [ + "GbL--" + ], + "split": [ + "GaLbb" + ] + }, + "userstyle": { + "unsplit": [ + "GbL--" + ], + "split": [ + "GaLbb" + ] + } + }, + "case_multivar_differ_noglobal": { + "input": "G-Lab", + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multivar_differ_diffglobal": { + "input": "GaLbc", + "localstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + } + }, + "case_multivar_differ_sameglobal": { + "input": "GaLab", + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + } + }, + "case_multivar_1none_noglobal": { + "input": "G-La-", + "localstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "userstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + } + }, + "case_multivar_1none_diffglobal": { + "input": "GaLb-", + "localstyle": { + "unsplit": [ + "G-Lba" + ], + "split": [ + "GaLb-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lba" + ], + "split": [ + "GaLb-" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lba" + ], + "split": [ + "GaLb-" + ] + } + }, + "case_multivar_1none_sameglobal": { + "input": "GaLa-", + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaLa-" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLa-" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLa-" + ] + } + }, + "case_multisource_gsame_lnone": { + "input": [ + "GaL-", + "GaL-" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaL--" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaL--" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaL--" + ] + } + }, + "case_multisource_gsame_lallsame": { + "input": [ + "GaLa", + "GaLa" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaLaa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLaa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLaa" + ] + } + }, + "case_multisource_gsame_l1same1none": { + "input": [ + "GaLa", + "GaL-" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaLa-" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLa-" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLa-" + ] + } + }, + "case_multisource_gsame_l1same1other": { + "input": [ + "GaLa", + "GaLb" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + } + }, + "case_multisource_gsame_lallother": { + "input": [ + "GaLb", + "GaLb" + ], + "localstyle": { + "unsplit": [ + "G-Lbb" + ], + "split": [ + "GaLbb" + ] + }, + "globalstyle": { + "unsplit": [ + "GbL--" + ], + "split": [ + "GaLbb" + ] + }, + "userstyle": { + "unsplit": [ + "GbL--" + ], + "split": [ + "GaLbb" + ] + } + }, + "case_multisource_gsame_lalldiffer": { + "input": [ + "GaLb", + "GaLc" + ], + "localstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + } + }, + "case_multisource_gnone_l1one1none": { + "input": [ + "G-La", + "G-L-" + ], + "localstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "userstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + } + }, + "case_multisource_gnone_l1one1same": { + "input": [ + "G-La", + "G-La" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "G-Laa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + } + }, + "case_multisource_gnone_l1one1other": { + "input": [ + "G-La", + "G-Lb" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_g1none_lnone": { + "input": [ + "GaL-", + "G-L-" + ], + "localstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "userstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + } + }, + "case_multisource_g1none_l1same1none": { + "input": [ + "GaLa", + "G-L-" + ], + "localstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "userstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + } + }, + "case_multisource_g1none_l1none1same": { + "input": [ + "GaL-", + "G-La" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "G-Laa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + } + }, + "case_multisource_g1none_l1diff1none": { + "input": [ + "GaLb", + "G-L-" + ], + "localstyle": { + "unsplit": [ + "G-Lb-" + ], + "split": [ + "G-Lb-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lb-" + ], + "split": [ + "G-Lb-" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lb-" + ], + "split": [ + "G-Lb-" + ] + } + }, + "case_multisource_g1none_l1none1diff": { + "input": [ + "GaL-", + "G-Lb" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_g1none_lallsame": { + "input": [ + "GaLa", + "G-La" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "G-Laa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + } + }, + "case_multisource_g1none_lallother": { + "input": [ + "GaLc", + "G-Lc" + ], + "localstyle": { + "unsplit": [ + "G-Lcc" + ], + "split": [ + "G-Lcc" + ] + }, + "globalstyle": { + "unsplit": [ + "GcL--" + ], + "split": [ + "G-Lcc" + ] + }, + "userstyle": { + "unsplit": [ + "GcL--" + ], + "split": [ + "G-Lcc" + ] + } + }, + "case_multisource_gdiff_lnone": { + "input": [ + "GaL-", + "GbL-" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_gdiff_l1same1none": { + "input": [ + "GaLa", + "GbL-" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_gdiff_l1diff1none": { + "input": [ + "GaLb", + "GcL-" + ], + "localstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "G-Lbc" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "G-Lbc" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "G-Lbc" + ] + } + }, + "case_multisource_gdiff_lallsame": { + "input": [ + "GaLa", + "GbLb" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_gdiff_lallother": { + "input": [ + "GaLc", + "GbLc" + ], + "localstyle": { + "unsplit": [ + "G-Lcc" + ], + "split": [ + "G-Lcc" + ] + }, + "globalstyle": { + "unsplit": [ + "GcL--" + ], + "split": [ + "G-Lcc" + ] + }, + "userstyle": { + "unsplit": [ + "GcL--" + ], + "split": [ + "G-Lcc" + ] + } + } +} \ No newline at end of file diff --git a/lib/iris/tests/integration/attrs_matrix_results_save.json b/lib/iris/tests/integration/attrs_matrix_results_save.json new file mode 100644 index 0000000000..3446c7f312 --- /dev/null +++ b/lib/iris/tests/integration/attrs_matrix_results_save.json @@ -0,0 +1,983 @@ +{ + "case_singlevar_localonly": { + "input": "G-La", + "localstyle": { + "unsplit": [ + "G-La" + ], + "split": [ + "G-La" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "G-La" + ] + }, + "userstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "G-La" + ] + } + }, + "case_singlevar_globalonly": { + "input": "GaL-", + "localstyle": { + "unsplit": [ + "G-La" + ], + "split": [ + "GaL-" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "GaL-" + ] + }, + "userstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "GaL-" + ] + } + }, + "case_singlevar_glsame": { + "input": "GaLa", + "localstyle": { + "unsplit": [ + "G-La" + ], + "split": [ + "GaLa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "GaLa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL-" + ], + "split": [ + "GaLa" + ] + } + }, + "case_singlevar_gldiffer": { + "input": "GaLb", + "localstyle": { + "unsplit": [ + "G-Lb" + ], + "split": [ + "GaLb" + ] + }, + "globalstyle": { + "unsplit": [ + "GbL-" + ], + "split": [ + "GaLb" + ] + }, + "userstyle": { + "unsplit": [ + "GbL-" + ], + "split": [ + "GaLb" + ] + } + }, + "case_multivar_same_noglobal": { + "input": "G-Laa", + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "G-Laa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + } + }, + "case_multivar_same_sameglobal": { + "input": "GaLaa", + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaLaa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLaa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLaa" + ] + } + }, + "case_multivar_same_diffglobal": { + "input": "GaLbb", + "localstyle": { + "unsplit": [ + "G-Lbb" + ], + "split": [ + "GaLbb" + ] + }, + "globalstyle": { + "unsplit": [ + "GbL--" + ], + "split": [ + "GaLbb" + ] + }, + "userstyle": { + "unsplit": [ + "GbL--" + ], + "split": [ + "GaLbb" + ] + } + }, + "case_multivar_differ_noglobal": { + "input": "G-Lab", + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multivar_differ_diffglobal": { + "input": "GaLbc", + "localstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + } + }, + "case_multivar_differ_sameglobal": { + "input": "GaLab", + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + } + }, + "case_multivar_1none_noglobal": { + "input": "G-La-", + "localstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "userstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + } + }, + "case_multivar_1none_diffglobal": { + "input": "GaLb-", + "localstyle": { + "unsplit": [ + "G-Lba" + ], + "split": [ + "GaLb-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lba" + ], + "split": [ + "GaLb-" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lba" + ], + "split": [ + "GaLb-" + ] + } + }, + "case_multivar_1none_sameglobal": { + "input": "GaLa-", + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaLa-" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLa-" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLa-" + ] + } + }, + "case_multisource_gsame_lnone": { + "input": [ + "GaL-", + "GaL-" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaL--" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaL--" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaL--" + ] + } + }, + "case_multisource_gsame_lallsame": { + "input": [ + "GaLa", + "GaLa" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaLaa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLaa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLaa" + ] + } + }, + "case_multisource_gsame_l1same1none": { + "input": [ + "GaLa", + "GaL-" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "GaLa-" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLa-" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "GaLa-" + ] + } + }, + "case_multisource_gsame_l1same1other": { + "input": [ + "GaLa", + "GaLb" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "GaLab" + ] + } + }, + "case_multisource_gsame_lallother": { + "input": [ + "GaLb", + "GaLb" + ], + "localstyle": { + "unsplit": [ + "G-Lbb" + ], + "split": [ + "GaLbb" + ] + }, + "globalstyle": { + "unsplit": [ + "GbL--" + ], + "split": [ + "GaLbb" + ] + }, + "userstyle": { + "unsplit": [ + "GbL--" + ], + "split": [ + "GaLbb" + ] + } + }, + "case_multisource_gsame_lalldiffer": { + "input": [ + "GaLb", + "GaLc" + ], + "localstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "GaLbc" + ] + } + }, + "case_multisource_gnone_l1one1none": { + "input": [ + "G-La", + "G-L-" + ], + "localstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "userstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + } + }, + "case_multisource_gnone_l1one1same": { + "input": [ + "G-La", + "G-La" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "G-Laa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + } + }, + "case_multisource_gnone_l1one1other": { + "input": [ + "G-La", + "G-Lb" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_g1none_lnone": { + "input": [ + "GaL-", + "G-L-" + ], + "localstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "userstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + } + }, + "case_multisource_g1none_l1same1none": { + "input": [ + "GaLa", + "G-L-" + ], + "localstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + }, + "userstyle": { + "unsplit": [ + "G-La-" + ], + "split": [ + "G-La-" + ] + } + }, + "case_multisource_g1none_l1none1same": { + "input": [ + "GaL-", + "G-La" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "G-Laa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + } + }, + "case_multisource_g1none_l1diff1none": { + "input": [ + "GaLb", + "G-L-" + ], + "localstyle": { + "unsplit": [ + "G-Lb-" + ], + "split": [ + "G-Lb-" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lb-" + ], + "split": [ + "G-Lb-" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lb-" + ], + "split": [ + "G-Lb-" + ] + } + }, + "case_multisource_g1none_l1none1diff": { + "input": [ + "GaL-", + "G-Lb" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_g1none_lallsame": { + "input": [ + "GaLa", + "G-La" + ], + "localstyle": { + "unsplit": [ + "G-Laa" + ], + "split": [ + "G-Laa" + ] + }, + "globalstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + }, + "userstyle": { + "unsplit": [ + "GaL--" + ], + "split": [ + "G-Laa" + ] + } + }, + "case_multisource_g1none_lallother": { + "input": [ + "GaLc", + "G-Lc" + ], + "localstyle": { + "unsplit": [ + "G-Lcc" + ], + "split": [ + "G-Lcc" + ] + }, + "globalstyle": { + "unsplit": [ + "GcL--" + ], + "split": [ + "G-Lcc" + ] + }, + "userstyle": { + "unsplit": [ + "GcL--" + ], + "split": [ + "G-Lcc" + ] + } + }, + "case_multisource_gdiff_lnone": { + "input": [ + "GaL-", + "GbL-" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_gdiff_l1same1none": { + "input": [ + "GaLa", + "GbL-" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_gdiff_l1diff1none": { + "input": [ + "GaLb", + "GcL-" + ], + "localstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "G-Lbc" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "G-Lbc" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lbc" + ], + "split": [ + "G-Lbc" + ] + } + }, + "case_multisource_gdiff_lallsame": { + "input": [ + "GaLa", + "GbLb" + ], + "localstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "globalstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + }, + "userstyle": { + "unsplit": [ + "G-Lab" + ], + "split": [ + "G-Lab" + ] + } + }, + "case_multisource_gdiff_lallother": { + "input": [ + "GaLc", + "GbLc" + ], + "localstyle": { + "unsplit": [ + "G-Lcc" + ], + "split": [ + "G-Lcc" + ] + }, + "globalstyle": { + "unsplit": [ + "GcL--" + ], + "split": [ + "G-Lcc" + ] + }, + "userstyle": { + "unsplit": [ + "GcL--" + ], + "split": [ + "G-Lcc" + ] + } + } +} \ No newline at end of file diff --git a/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py b/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py index a1cad53336..9bd996312c 100644 --- a/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py +++ b/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py @@ -19,13 +19,20 @@ """ import inspect -from typing import Iterable, Optional, Union - +import json +import os +from pathlib import Path +import re +from typing import Iterable, List, Optional, Union +import warnings + +import numpy as np import pytest import iris import iris.coord_systems -from iris.cube import Cube, CubeAttrsDict +from iris.coords import DimCoord +from iris.cube import Cube import iris.fileformats.netcdf import iris.fileformats.netcdf._thread_safe_nc as threadsafe_nc4 @@ -39,16 +46,18 @@ # A list of "global-style" attribute names : those which should be global attributes by # default (i.e. file- or group-level, *not* attached to a variable). + _GLOBAL_TEST_ATTRS = set(iris.fileformats.netcdf.saver._CF_GLOBAL_ATTRS) # Remove this one, which has peculiar behaviour + is tested separately # N.B. this is not the same as 'Conventions', but is caught in the crossfire when that # one is processed. _GLOBAL_TEST_ATTRS -= set(["conventions"]) +_GLOBAL_TEST_ATTRS = sorted(_GLOBAL_TEST_ATTRS) # Define a fixture to parametrise tests over the 'global-style' test attributes. # This just provides a more concise way of writing parametrised tests. -@pytest.fixture(params=sorted(_GLOBAL_TEST_ATTRS)) +@pytest.fixture(params=_GLOBAL_TEST_ATTRS) def global_attr(request): # N.B. "request" is a standard PyTest fixture return request.param # Return the name of the attribute to test. @@ -64,12 +73,63 @@ def global_attr(request): # Define a fixture to parametrise over the 'local-style' test attributes. # This just provides a more concise way of writing parametrised tests. -@pytest.fixture(params=sorted(_LOCAL_TEST_ATTRS)) +@pytest.fixture(params=_LOCAL_TEST_ATTRS) def local_attr(request): # N.B. "request" is a standard PyTest fixture return request.param # Return the name of the attribute to test. +# Define whether to parametrise over split-attribute saving +# Just for now, so that we can run against legacy code. +_SPLIT_SAVE_SUPPORTED = hasattr(iris.FUTURE, "save_split_attrs") +_SPLIT_PARAM_VALUES = [False, True] +_SPLIT_PARAM_IDS = ["nosplit", "split"] +_MATRIX_LOAD_RESULTSTYLES = ["legacy", "newstyle"] +if not _SPLIT_SAVE_SUPPORTED: + _SPLIT_PARAM_VALUES.remove(True) + _SPLIT_PARAM_IDS.remove("split") + _MATRIX_LOAD_RESULTSTYLES.remove("newstyle") + + +_SKIP_WARNCHECK = "_no_warnings_check" + + +def check_captured_warnings( + expected_keys: List[str], captured_warnings: List[warnings.WarningMessage] +): + """ + Compare captured warning messages with a list of regexp-matches. + + We allow them to occur in any order, and replace each actual result in the list + with its matching regexp, if any, as this makes failure results much easier to + comprehend. + + """ + if expected_keys is None: + expected_keys = [] + elif hasattr(expected_keys, "upper"): + # Handle a single string + if expected_keys == _SKIP_WARNCHECK: + # No check at all in this case + return + expected_keys = [expected_keys] + + expected_keys = [re.compile(key) for key in expected_keys] + found_results = [str(warning.message) for warning in captured_warnings] + remaining_keys = expected_keys.copy() + for i_message, message in enumerate(found_results.copy()): + for key in remaining_keys: + if key.search(message): + # Hit : replace one message in the list with its matching "key" + found_results[i_message] = key + # remove the matching key + remaining_keys.remove(key) + # skip on to next message + break + + assert set(found_results) == set(expected_keys) + + class MixinAttrsTesting: @staticmethod def _calling_testname(): @@ -77,7 +137,7 @@ def _calling_testname(): Search up the callstack for a function named "test_*", and return the name for use as a test identifier. - Idea borrowed from :meth:`iris.tests.IrisTest_nometa.result_path`. + Idea borrowed from :meth:`iris.tests.IrisTest.result_path`. Returns ------- @@ -128,28 +188,34 @@ def _default_vars_and_attrvalues(vars_and_attrvalues): vars_and_attrvalues = {"var": vars_and_attrvalues} return vars_and_attrvalues - def create_testcase_files( + def create_testcase_files_or_cubes( self, attr_name: str, global_value_file1: Optional[str] = None, var_values_file1: Union[None, str, dict] = None, global_value_file2: Optional[str] = None, var_values_file2: Union[None, str, dict] = None, + cubes: bool = False, ): """ - Create temporary input netcdf files with specific content. + Create temporary input netcdf files, or cubes, with specific content. Creates a temporary netcdf test file (or two) with the given global and - variable-local attributes. - The file(s) are used to test the behaviour of the attribute. + variable-local attributes. Or build cubes, similarly. + If ``cubes`` is ``True``, save cubes in ``self.input_cubes``. + Else save filepaths in ``self.input_filepaths``. Note: 'var_values_file' args are dictionaries. The named variables are created, with an attribute = the dictionary value, *except* that a dictionary value of None means that a local attribute is _not_ created on the variable. """ - # Make some input file paths. - filepath1 = self._testfile_path("testfile") - filepath2 = self._testfile_path("testfile2") + # save attribute on the instance + self.attrname = attr_name + + if not cubes: + # Make some input file paths. + filepath1 = self._testfile_path("testfile") + filepath2 = self._testfile_path("testfile2") def make_file( filepath: str, global_value=None, var_values=None @@ -169,24 +235,424 @@ def make_file( ds.close() return filepath - # Create one input file (always). - filepaths = [ - make_file( - filepath1, - global_value=global_value_file1, - var_values=var_values_file1, - ) - ] - if global_value_file2 is not None or var_values_file2 is not None: - # Make a second testfile and add it to files-to-be-loaded. - filepaths.append( - make_file( - filepath2, - global_value=global_value_file2, - var_values=var_values_file2, - ), + def make_cubes(var_name, global_value=None, var_values=None): + cubes = [] + var_values = self._default_vars_and_attrvalues(var_values) + for varname, local_value in var_values.items(): + cube = Cube(np.arange(3.0), var_name=var_name) + cubes.append(cube) + dimco = DimCoord(np.arange(3.0), var_name="x") + cube.add_dim_coord(dimco, 0) + if not hasattr(cube.attributes, "globals"): + # N.B. For now, also support oldstyle "single" cube attribute + # dictionaries, so that we can generate legacy results to compore + # with the "new world" results. + single_value = global_value + if local_value is not None: + single_value = local_value + if single_value is not None: + cube.attributes[attr_name] = single_value + else: + if global_value is not None: + cube.attributes.globals[attr_name] = global_value + if local_value is not None: + cube.attributes.locals[attr_name] = local_value + return cubes + + if cubes: + results = make_cubes("v1", global_value_file1, var_values_file1) + if global_value_file2 is not None or var_values_file2 is not None: + results.extend( + make_cubes("v2", global_value_file2, var_values_file2) + ) + else: + results = [ + make_file(filepath1, global_value_file1, var_values_file1) + ] + if global_value_file2 is not None or var_values_file2 is not None: + # Make a second testfile and add it to files-to-be-loaded. + results.append( + make_file(filepath2, global_value_file2, var_values_file2) + ) + + # Save results on the instance + if cubes: + self.input_cubes = results + else: + self.input_filepaths = results + return results + + def run_testcase( + self, + attr_name: str, + values: Union[List, List[List]], + create_cubes_or_files: str = "files", + ) -> None: + """ + Create testcase inputs (files or cubes) with specified attributes. + + Parameters + ---------- + attr_name : str + name for all attributes created in this testcase. + Also saved as ``self.attrname``, as used by ``fetch_results``. + values : list + list, or lists, of values for created attributes, each containing one global + and one-or-more local attribute values as [global, local1, local2...] + create_cubes_or_files : str, default "files" + create either cubes or testfiles. + + If ``create_cubes_or_files`` == "files", create one temporary netCDF file per + values-list, and record in ``self.input_filepaths``. + Else if ``create_cubes_or_files`` == "cubes", create sets of cubes with common + global values and store all of them to ``self.input_cubes``. + + """ + # Save common attribute-name on the instance + self.attrname = attr_name + + # Standardise input to a list-of-lists, each inner list = [global, *locals] + assert isinstance(values, list) + if not isinstance(values[0], list): + values = [values] + assert len(values) in (1, 2) + assert len(values[0]) > 1 + + # Decode into global1, *locals1, and optionally global2, *locals2 + global1 = values[0][0] + vars1 = {} + i_var = 0 + for value in values[0][1:]: + vars1[f"var_{i_var}"] = value + i_var += 1 + if len(values) == 1: + global2 = None + vars2 = None + else: + assert len(values) == 2 + global2 = values[1][0] + vars2 = {} + for value in values[1][1:]: + vars2[f"var_{i_var}"] = value + i_var += 1 + + # Create test files or cubes (and store data on the instance) + assert create_cubes_or_files in ("cubes", "files") + make_cubes = create_cubes_or_files == "cubes" + self.create_testcase_files_or_cubes( + attr_name=attr_name, + global_value_file1=global1, + var_values_file1=vars1, + global_value_file2=global2, + var_values_file2=vars2, + cubes=make_cubes, + ) + + def fetch_results( + self, + filepath: str = None, + cubes: Iterable[Cube] = None, + oldstyle_combined: bool = False, + ): + """ + Return testcase results from an output file or cubes in a standardised form. + + Unpick the global+local values of the attribute ``self.attrname``, resulting + from a test operation. + A file result is always [global_value, *local_values] + A cubes result is [*[global_value, *local_values]] (over different global vals) + + When ``oldstyle_combined`` is ``True``, simulate the "legacy" style results, + that is when each cube had a single combined attribute dictionary. + This enables us to check against former behaviour, by combining results into a + single dictionary. N.B. per-cube single results are then returned in the form: + [None, cube1, cube2...]. + N.B. if results are from a *file*, this key has **no effect**. + + """ + attr_name = self.attrname + if filepath is not None: + # Fetch global and local values from a file + try: + ds = threadsafe_nc4.DatasetWrapper(filepath) + global_result = ( + ds.getncattr(attr_name) + if attr_name in ds.ncattrs() + else None + ) + # Fetch local attr value from all data variables : In our testcases, + # that is all *except* dimcoords (ones named after dimensions). + local_vars_results = [ + ( + var.name, + ( + var.getncattr(attr_name) + if attr_name in var.ncattrs() + else None + ), + ) + for var in ds.variables.values() + if var.name not in ds.dimensions + ] + finally: + ds.close() + # This version always returns a single result set [global, local1[, local2]] + # Return global, plus locals sorted by varname + local_vars_results = sorted(local_vars_results, key=lambda x: x[0]) + results = [global_result] + [val for _, val in local_vars_results] + else: + assert cubes is not None + # Sort result cubes according to a standard ordering. + cubes = sorted(cubes, key=lambda cube: cube.name()) + # Fetch globals and locals from cubes. + # This way returns *multiple* result 'sets', one for each global value + if oldstyle_combined or not _SPLIT_SAVE_SUPPORTED: + # Use all-combined dictionaries in place of actual cubes' attributes + cube_attr_dicts = [dict(cube.attributes) for cube in cubes] + # Return results as if all cubes had global=None + results = [ + [None] + + [ + cube_attr_dict.get(attr_name, None) + for cube_attr_dict in cube_attr_dicts + ] + ] + else: + # Return a result-set for each occurring global value (possibly + # including a 'None'). + global_values = set( + cube.attributes.globals.get(attr_name, None) + for cube in cubes + ) + results = [ + [globalval] + + [ + cube.attributes.locals.get(attr_name, None) + for cube in cubes + if cube.attributes.globals.get(attr_name, None) + == globalval + ] + for globalval in sorted(global_values, key=str) + ] + return results + + +# Define all the testcases for different parameter input structures : +# - combinations of matching+differing, global+local params +# - these are interpreted differently for the 3 main test types : Load/Save/Roundtrip +_MATRIX_TESTCASE_INPUTS = { + "case_singlevar_localonly": "G-La", + "case_singlevar_globalonly": "GaL-", + "case_singlevar_glsame": "GaLa", + "case_singlevar_gldiffer": "GaLb", + "case_multivar_same_noglobal": "G-Laa", + "case_multivar_same_sameglobal": "GaLaa", + "case_multivar_same_diffglobal": "GaLbb", + "case_multivar_differ_noglobal": "G-Lab", + "case_multivar_differ_diffglobal": "GaLbc", + "case_multivar_differ_sameglobal": "GaLab", + "case_multivar_1none_noglobal": "G-La-", + "case_multivar_1none_diffglobal": "GaLb-", + "case_multivar_1none_sameglobal": "GaLa-", + # Note: the multi-set input cases are more complex. + # These are encoded as *pairs* of specs, for 2 different files, or cubes with + # independent global values. + # We assume that there can be nothing "special" about a var's interaction with + # another one from the same (as opposed to the "other") file. + "case_multisource_gsame_lnone": ["GaL-", "GaL-"], + "case_multisource_gsame_lallsame": ["GaLa", "GaLa"], + "case_multisource_gsame_l1same1none": ["GaLa", "GaL-"], + "case_multisource_gsame_l1same1other": ["GaLa", "GaLb"], + "case_multisource_gsame_lallother": ["GaLb", "GaLb"], + "case_multisource_gsame_lalldiffer": ["GaLb", "GaLc"], + "case_multisource_gnone_l1one1none": ["G-La", "G-L-"], + "case_multisource_gnone_l1one1same": ["G-La", "G-La"], + "case_multisource_gnone_l1one1other": ["G-La", "G-Lb"], + "case_multisource_g1none_lnone": ["GaL-", "G-L-"], + "case_multisource_g1none_l1same1none": ["GaLa", "G-L-"], + "case_multisource_g1none_l1none1same": ["GaL-", "G-La"], + "case_multisource_g1none_l1diff1none": ["GaLb", "G-L-"], + "case_multisource_g1none_l1none1diff": ["GaL-", "G-Lb"], + "case_multisource_g1none_lallsame": ["GaLa", "G-La"], + "case_multisource_g1none_lallother": ["GaLc", "G-Lc"], + "case_multisource_gdiff_lnone": ["GaL-", "GbL-"], + "case_multisource_gdiff_l1same1none": ["GaLa", "GbL-"], + "case_multisource_gdiff_l1diff1none": ["GaLb", "GcL-"], + "case_multisource_gdiff_lallsame": ["GaLa", "GbLb"], + "case_multisource_gdiff_lallother": ["GaLc", "GbLc"], +} +_MATRIX_TESTCASES = list(_MATRIX_TESTCASE_INPUTS.keys()) + +# +# Define the attrs against which all matrix tests are run +# +max_param_attrs = None +# max_param_attrs = 5 + +_MATRIX_ATTRNAMES = _LOCAL_TEST_ATTRS[:max_param_attrs] +_MATRIX_ATTRNAMES += _GLOBAL_TEST_ATTRS[:max_param_attrs] +_MATRIX_ATTRNAMES += ["user"] + +# remove special-cases, for now : all these behave irregularly (i.e. unlike the known +# "globalstyle", or "localstyle" generic cases). +# N.B. not including "Conventions", which is not in the globals list, so won't be +# matrix-tested unless we add it specifically. +# TODO: decide if any of these need to be tested, as separate test-styles. +_SPECIAL_ATTRS = [ + "ukmo__process_flags", + "missing_value", + "standard_error_multiplier", + "STASH", + "um_stash_source", +] +_MATRIX_ATTRNAMES = [ + attr for attr in _MATRIX_ATTRNAMES if attr not in _SPECIAL_ATTRS +] + + +# +# A routine to work "backwards" from an attribute name to its "style", i.e. type category. +# Possible styles are "globalstyle", "localstyle", "userstyle". +# +_ATTR_STYLES = ["localstyle", "globalstyle", "userstyle"] + + +def deduce_attr_style(attrname: str) -> str: + # Extract the attribute "style type" from an attr_param name + if attrname in _LOCAL_TEST_ATTRS: + style = "localstyle" + elif attrname in _GLOBAL_TEST_ATTRS: + style = "globalstyle" + else: + assert attrname == "user" + style = "userstyle" + return style + + +# +# Decode a matrix "input spec" to codes for global + local values. +# +def decode_matrix_input(input_spec): + # Decode a matrix-test input specification, like "GaLbc", into lists of values. + # E.G. "GaLbc" -> ["a", "b", "c"] + # ["GaLbc", "GbLbc"] -> [["a", "b", "c"], ["b", "b", c"]] + # N.B. in this form "values" are all one-character strings. + def decode_specstring(spec: str) -> List[Union[str, None]]: + # Decode an input spec-string to input/output attribute values + assert spec[0] == "G" and spec[2] == "L" + allvals = spec[1] + spec[3:] + result = [None if valchar == "-" else valchar for valchar in allvals] + return result + + if isinstance(input_spec, str): + # Single-source spec (one cube or one file) + vals = decode_specstring(input_spec) + result = [vals] + else: + # Dual-source spec (two files, or sets of cubes with a common global value) + vals_A = decode_specstring(input_spec[0]) + vals_B = decode_specstring(input_spec[1]) + result = [vals_A, vals_B] + + return result + + +def encode_matrix_result(results: List[List[str]]) -> List[str]: + # Re-code a set of output results, [*[global-value, *local-values]] as a list of + # strings, like ["GaL-b"] or ["GaLabc", "GbLabc"]. + # N.B. again assuming that all values are just one-character strings, or None. + assert isinstance(results, Iterable) and len(results) >= 1 + if not isinstance(results[0], list): + results = [results] + assert all( + all(val is None or len(val) == 1 for val in vals) for vals in results + ) + + # Translate "None" values to "-" + def valrep(val): + return "-" if val is None else val + + results = list( + "".join(["G", valrep(vals[0]), "L"] + list(map(valrep, vals[1:]))) + for vals in results + ) + return results + + +# +# The "expected" matrix test results are stored in JSON files (one for each test-type). +# We can also save the found results. +# +_MATRIX_TESTTYPES = ("load", "save", "roundtrip") + + +@pytest.fixture(autouse=True, scope="session") +def matrix_results(): + matrix_filepaths = { + testtype: ( + Path(__file__).parent / f"attrs_matrix_results_{testtype}.json" + ) + for testtype in _MATRIX_TESTTYPES + } + # An environment variable can trigger saving of the results. + save_matrix_results = bool( + int(os.environ.get("SAVEALL_MATRIX_RESULTS", "0")) + ) + + matrix_results = {} + for testtype in _MATRIX_TESTTYPES: + # Either fetch from file, or initialise, a results matrix for each test type + # (load/save/roundtrip). + input_path = matrix_filepaths[testtype] + if input_path.exists(): + # Load from file with json. + with open(input_path) as file_in: + testtype_results = json.load(file_in) + # Check compatibility (in case we changed the test-specs list) + assert set(testtype_results.keys()) == set(_MATRIX_TESTCASES) + assert all( + testtype_results[key]["input"] == _MATRIX_TESTCASE_INPUTS[key] + for key in _MATRIX_TESTCASES ) - return filepaths + else: + # Create empty matrix results content (for one test-type) + testtype_results = {} + for testcase in _MATRIX_TESTCASES: + test_case_results = {} + testtype_results[testcase] = test_case_results + # Every testcase dict has an "input" slot with the test input spec, + # basically just to help human readability. + test_case_results["input"] = _MATRIX_TESTCASE_INPUTS[testcase] + for attrstyle in _ATTR_STYLES: + if testtype == "load": + # "load" test results have a "legacy" result (as for a single + # combined attrs dictionary), and a "newstyle" result (with + # the new split dictionary). + test_case_results[attrstyle] = { + "legacy": None, + "newstyle": None, + } + else: + # "save"/"roundtrip"-type results record 2 result sets, + # (unsplit/split) for each attribute-style + # - i.e. when saved without/with split_attrs_saving enabled. + test_case_results[attrstyle] = { + "unsplit": None, + "split": None, + } + + # Build complete data: matrix_results[TESTTYPES][TESTCASES][ATTR_STYLES] + matrix_results[testtype] = testtype_results + + # Pass through to all the tests : they can also update it, if enabled. + yield save_matrix_results, matrix_results + + if save_matrix_results: + for testtype in _MATRIX_TESTTYPES: + output_path = matrix_filepaths[testtype] + results = matrix_results[testtype] + with open(output_path, "w") as file_out: + json.dump(results, file_out, indent=2) class TestRoundtrip(MixinAttrsTesting): @@ -205,24 +671,16 @@ class TestRoundtrip(MixinAttrsTesting): """ - def _roundtrip_load_and_save( - self, input_filepaths: Union[str, Iterable[str]], output_filepath: str - ) -> None: - """ - Load netcdf input file(s) and re-write all to a given output file. - """ - # Do a load+save to produce a testable output result in a new file. - cubes = iris.load(input_filepaths) - iris.save(cubes, output_filepath) + # Parametrise all tests over split/unsplit saving. + @pytest.fixture( + params=_SPLIT_PARAM_VALUES, ids=_SPLIT_PARAM_IDS, autouse=True + ) + def do_split(self, request): + do_split = request.param + self.save_split_attrs = do_split + return do_split - def create_roundtrip_testcase( - self, - attr_name, - global_value_file1=None, - vars_values_file1=None, - global_value_file2=None, - vars_values_file2=None, - ): + def run_roundtrip_testcase(self, attr_name, values): """ Initialise the testcase from the passed-in controls, configure the input files and run a save-load roundtrip to produce the output file. @@ -231,48 +689,44 @@ def create_roundtrip_testcase( stored on the instance, where "self.check_roundtrip_results()" can get them. """ - self.attrname = attr_name - self.input_filepaths = self.create_testcase_files( - attr_name=attr_name, - global_value_file1=global_value_file1, - var_values_file1=vars_values_file1, - global_value_file2=global_value_file2, - var_values_file2=vars_values_file2, + self.run_testcase( + attr_name=attr_name, values=values, create_cubes_or_files="files" ) self.result_filepath = self._testfile_path("result") - self._roundtrip_load_and_save( - self.input_filepaths, self.result_filepath - ) - def check_roundtrip_results( - self, global_attr_value=None, var_attr_vals=None - ): + with warnings.catch_warnings(record=True) as captured_warnings: + # Do a load+save to produce a testable output result in a new file. + cubes = iris.load(self.input_filepaths) + # Ensure stable result order. + cubes = sorted(cubes, key=lambda cube: cube.name()) + do_split = getattr(self, "save_split_attrs", False) + kwargs = ( + dict(save_split_attrs=do_split) + if _SPLIT_SAVE_SUPPORTED + else dict() + ) + with iris.FUTURE.context(**kwargs): + iris.save(cubes, self.result_filepath) + + self.captured_warnings = captured_warnings + + def check_roundtrip_results(self, expected, expected_warnings=None): """ Run checks on the generated output file. - The counterpart to create_testcase, with similar control arguments. - Check existence (or not) of : a global attribute, named variables, and their - local attributes. Values of 'None' mean to check that the relevant global/local - attribute does *not* exist. + The counterpart to :meth:`run_roundtrip_testcase`, with similar arguments. + Check existence (or not) of a global attribute, and a number of local + (variable) attributes. + Values of 'None' mean to check that the relevant global/local attribute does + *not* exist. + + Also check the warnings captured during the testcase run. """ # N.B. there is only ever one result-file, but it can contain various variables # which came from different input files. - ds = threadsafe_nc4.DatasetWrapper(self.result_filepath) - if global_attr_value is None: - assert self.attrname not in ds.ncattrs() - else: - assert self.attrname in ds.ncattrs() - assert ds.getncattr(self.attrname) == global_attr_value - if var_attr_vals: - var_attr_vals = self._default_vars_and_attrvalues(var_attr_vals) - for var_name, value in var_attr_vals.items(): - assert var_name in ds.variables - v = ds.variables[var_name] - if value is None: - assert self.attrname not in v.ncattrs() - else: - assert self.attrname in v.ncattrs() - assert v.getncattr(self.attrname) == value + results = self.fetch_results(filepath=self.result_filepath) + assert results == expected + check_captured_warnings(expected_warnings, self.captured_warnings) ####################################################### # Tests on "user-style" attributes. @@ -281,94 +735,100 @@ def check_roundtrip_results( # def test_01_userstyle_single_global(self): - self.create_roundtrip_testcase( - attr_name="myname", # A generic "user" attribute with no special handling - global_value_file1="single-value", - vars_values_file1={ - "myvar": None - }, # the variable has no such attribute + self.run_roundtrip_testcase( + attr_name="myname", values=["single-value", None] ) # Default behaviour for a general global user-attribute. # It simply remains global. - self.check_roundtrip_results( - global_attr_value="single-value", # local values eclipse the global ones - var_attr_vals={ - "myvar": None - }, # the variable has no such attribute - ) + self.check_roundtrip_results(["single-value", None]) - def test_02_userstyle_single_local(self): + def test_02_userstyle_single_local(self, do_split): # Default behaviour for a general local user-attribute. # It results in a "promoted" global attribute. - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name="myname", # A generic "user" attribute with no special handling - vars_values_file1={"myvar": "single-value"}, - ) - self.check_roundtrip_results( - global_attr_value="single-value", # local values eclipse the global ones - # N.B. the output var has NO such attribute + values=[None, "single-value"], ) + if do_split: + expected = [None, "single-value"] + else: + expected = ["single-value", None] + self.check_roundtrip_results(expected) - def test_03_userstyle_multiple_different(self): + def test_03_userstyle_multiple_different(self, do_split): # Default behaviour for general user-attributes. # The global attribute is lost because there are local ones. - vars1 = {"f1_v1": "f1v1", "f1_v2": "f2v2"} - vars2 = {"f2_v1": "x1", "f2_v2": "x2"} - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name="random", # A generic "user" attribute with no special handling - global_value_file1="global_file1", - vars_values_file1=vars1, - global_value_file2="global_file2", - vars_values_file2=vars2, - ) - # combine all 4 vars in one dict - all_vars_and_attrs = vars1.copy() - all_vars_and_attrs.update(vars2) - # TODO: replace with "|", when we drop Python 3.8 - # see: https://peps.python.org/pep-0584/ - # just check they are all there and distinct - assert len(all_vars_and_attrs) == len(vars1) + len(vars2) - self.check_roundtrip_results( - global_attr_value=None, # local values eclipse the global ones - var_attr_vals=all_vars_and_attrs, + values=[ + ["common_global", "f1v1", "f1v2"], + ["common_global", "x1", "x2"], + ], ) + expected_result = ["common_global", "f1v1", "f1v2", "x1", "x2"] + if not do_split: + # in legacy mode, global is lost + expected_result[0] = None + # just check they are all there and distinct + self.check_roundtrip_results(expected_result) - def test_04_userstyle_matching_promoted(self): + def test_04_userstyle_matching_promoted(self, do_split): # matching local user-attributes are "promoted" to a global one. - self.create_roundtrip_testcase( + # (but not when saving split attributes) + input_values = ["global_file1", "same-value", "same-value"] + self.run_roundtrip_testcase( attr_name="random", - global_value_file1="global_file1", - vars_values_file1={"v1": "same-value", "v2": "same-value"}, - ) - self.check_roundtrip_results( - global_attr_value="same-value", - var_attr_vals={"v1": None, "v2": None}, + values=input_values, ) + if do_split: + expected = input_values + else: + expected = ["same-value", None, None] + self.check_roundtrip_results(expected) - def test_05_userstyle_matching_crossfile_promoted(self): + def test_05_userstyle_matching_crossfile_promoted(self, do_split): # matching user-attributes are promoted, even across input files. - self.create_roundtrip_testcase( + # (but not when saving split attributes) + self.run_roundtrip_testcase( attr_name="random", - global_value_file1="global_file1", - vars_values_file1={"v1": "same-value", "v2": "same-value"}, - vars_values_file2={"f2_v1": "same-value", "f2_v2": "same-value"}, - ) - self.check_roundtrip_results( - global_attr_value="same-value", - var_attr_vals={x: None for x in ("v1", "v2", "f2_v1", "f2_v2")}, + values=[ + ["global_file1", "same-value", "same-value"], + [None, "same-value", "same-value"], + ], ) + if do_split: + # newstyle saves: locals are preserved, mismathced global is *lost* + expected_result = [ + None, + "same-value", + "same-value", + "same-value", + "same-value", + ] + # warnings about the clash + expected_warnings = [ + "Saving.* global attributes.* as local", + 'attributes.* of cube "var_0" were not saved', + 'attributes.* of cube "var_1" were not saved', + ] + else: + # oldstyle saves: matching locals promoted, override original global + expected_result = ["same-value", None, None, None, None] + expected_warnings = None - def test_06_userstyle_nonmatching_remainlocal(self): + self.check_roundtrip_results(expected_result, expected_warnings) + + def test_06_userstyle_nonmatching_remainlocal(self, do_split): # Non-matching user attributes remain 'local' to the individual variables. - self.create_roundtrip_testcase( - attr_name="random", - global_value_file1="global_file1", - vars_values_file1={"v1": "value-1", "v2": "value-2"}, - ) - self.check_roundtrip_results( - global_attr_value=None, # NB it still destroys the global one !! - var_attr_vals={"v1": "value-1", "v2": "value-2"}, - ) + input_values = ["global_file1", "value-1", "value-2"] + if do_split: + # originals are preserved + expected_result = input_values + else: + # global is lost + expected_result = [None, "value-1", "value-2"] + self.run_roundtrip_testcase(attr_name="random", values=input_values) + self.check_roundtrip_results(expected_result) ####################################################### # Tests on "Conventions" attribute. @@ -383,129 +843,121 @@ def test_06_userstyle_nonmatching_remainlocal(self): def test_07_conventions_var_local(self): # What happens if 'Conventions' appears as a variable-local attribute. # N.B. this is not good CF, but we'll see what happens anyway. - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name="Conventions", - global_value_file1=None, - vars_values_file1="user_set", - ) - self.check_roundtrip_results( - global_attr_value="CF-1.7", # standard content from Iris save - var_attr_vals=None, + values=[None, "user_set"], ) + self.check_roundtrip_results(["CF-1.7", None]) def test_08_conventions_var_both(self): # What happens if 'Conventions' appears as both global + local attribute. - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name="Conventions", - global_value_file1="global-setting", - vars_values_file1="local-setting", - ) - self.check_roundtrip_results( - global_attr_value="CF-1.7", # standard content from Iris save - var_attr_vals=None, + values=["global-setting", "local-setting"], ) + # standard content from Iris save + self.check_roundtrip_results(["CF-1.7", None]) ####################################################### # Tests on "global" style attributes # = those specific ones which 'ought' only to be global (except on collisions) # - def test_09_globalstyle__global(self, global_attr): attr_content = f"Global tracked {global_attr}" - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name=global_attr, - global_value_file1=attr_content, + values=[attr_content, None], ) - self.check_roundtrip_results(global_attr_value=attr_content) + self.check_roundtrip_results([attr_content, None]) - def test_10_globalstyle__local(self, global_attr): + def test_10_globalstyle__local(self, global_attr, do_split): # Strictly, not correct CF, but let's see what it does with it. attr_content = f"Local tracked {global_attr}" - self.create_roundtrip_testcase( + input_values = [None, attr_content] + self.run_roundtrip_testcase( attr_name=global_attr, - vars_values_file1=attr_content, + values=input_values, ) - self.check_roundtrip_results( - global_attr_value=attr_content - ) # "promoted" + if do_split: + # remains local as supplied, but there is a warning + expected_result = input_values + expected_warning = f"'{global_attr}'.* should only be a CF global" + else: + # promoted to global + expected_result = [attr_content, None] + expected_warning = None + self.check_roundtrip_results(expected_result, expected_warning) - def test_11_globalstyle__both(self, global_attr): + def test_11_globalstyle__both(self, global_attr, do_split): attr_global = f"Global-{global_attr}" attr_local = f"Local-{global_attr}" - self.create_roundtrip_testcase( + input_values = [attr_global, attr_local] + self.run_roundtrip_testcase( attr_name=global_attr, - global_value_file1=attr_global, - vars_values_file1=attr_local, - ) - self.check_roundtrip_results( - global_attr_value=attr_local # promoted local setting "wins" + values=input_values, ) + if do_split: + # remains local as supplied, but there is a warning + expected_result = input_values + expected_warning = "should only be a CF global" + else: + # promoted to global, no local value, original global lost + expected_result = [attr_local, None] + expected_warning = None + self.check_roundtrip_results(expected_result, expected_warning) def test_12_globalstyle__multivar_different(self, global_attr): # Multiple *different* local settings are retained, not promoted attr_1 = f"Local-{global_attr}-1" attr_2 = f"Local-{global_attr}-2" - with pytest.warns( - UserWarning, match="should only be a CF global attribute" - ): - # A warning should be raised when writing the result. - self.create_roundtrip_testcase( - attr_name=global_attr, - vars_values_file1={"v1": attr_1, "v2": attr_2}, - ) - self.check_roundtrip_results( - global_attr_value=None, - var_attr_vals={"v1": attr_1, "v2": attr_2}, + expect_warning = "should only be a CF global attribute" + # A warning should be raised when writing the result. + self.run_roundtrip_testcase( + attr_name=global_attr, + values=[None, attr_1, attr_2], ) + self.check_roundtrip_results([None, attr_1, attr_2], expect_warning) - def test_13_globalstyle__multivar_same(self, global_attr): + def test_13_globalstyle__multivar_same(self, global_attr, do_split): # Multiple *same* local settings are promoted to a common global one attrval = f"Locally-defined-{global_attr}" - self.create_roundtrip_testcase( + input_values = [None, attrval, attrval] + self.run_roundtrip_testcase( attr_name=global_attr, - vars_values_file1={"v1": attrval, "v2": attrval}, - ) - self.check_roundtrip_results( - global_attr_value=attrval, - var_attr_vals={"v1": None, "v2": None}, + values=input_values, ) + if do_split: + # remains local, but with a warning + expected_warning = "should only be a CF global" + expected_result = input_values + else: + # promoted to global + expected_warning = None + expected_result = [attrval, None, None] + self.check_roundtrip_results(expected_result, expected_warning) - def test_14_globalstyle__multifile_different(self, global_attr): + def test_14_globalstyle__multifile_different(self, global_attr, do_split): # Different global attributes from multiple files are retained as local ones attr_1 = f"Global-{global_attr}-1" attr_2 = f"Global-{global_attr}-2" - with pytest.warns( - UserWarning, match="should only be a CF global attribute" - ): - # A warning should be raised when writing the result. - self.create_roundtrip_testcase( - attr_name=global_attr, - global_value_file1=attr_1, - vars_values_file1={"v1": None}, - global_value_file2=attr_2, - vars_values_file2={"v2": None}, - ) - self.check_roundtrip_results( - # Combining them "demotes" the common global attributes to local ones - var_attr_vals={"v1": attr_1, "v2": attr_2} + self.run_roundtrip_testcase( + attr_name=global_attr, + values=[[attr_1, None], [attr_2, None]], ) + # A warning should be raised when writing the result. + expected_warnings = ["should only be a CF global attribute"] + if do_split: + # An extra warning, only when saving with split-attributes. + expected_warnings = ["Saving.* as local"] + expected_warnings + self.check_roundtrip_results([None, attr_1, attr_2], expected_warnings) def test_15_globalstyle__multifile_same(self, global_attr): # Matching global-type attributes in multiple files are retained as global attrval = f"Global-{global_attr}" - self.create_roundtrip_testcase( - attr_name=global_attr, - global_value_file1=attrval, - vars_values_file1={"v1": None}, - global_value_file2=attrval, - vars_values_file2={"v2": None}, - ) - self.check_roundtrip_results( - # The attribute remains as a common global setting - global_attr_value=attrval, - # The individual variables do *not* have an attribute of this name - var_attr_vals={"v1": None, "v2": None}, + self.run_roundtrip_testcase( + attr_name=global_attr, values=[[attrval, None], [attrval, None]] ) + self.check_roundtrip_results([attrval, None, None]) ####################################################### # Tests on "local" style attributes @@ -514,7 +966,7 @@ def test_15_globalstyle__multifile_same(self, global_attr): # @pytest.mark.parametrize("origin_style", ["input_global", "input_local"]) - def test_16_localstyle(self, local_attr, origin_style): + def test_16_localstyle(self, local_attr, origin_style, do_split): # local-style attributes should *not* get 'promoted' to global ones # Set the name extension to avoid tests with different 'style' params having # collisions over identical testfile names @@ -535,20 +987,19 @@ def test_16_localstyle(self, local_attr, origin_style): # as global or a variable attribute if origin_style == "input_global": # Record in source as a global attribute - self.create_roundtrip_testcase( - attr_name=local_attr, global_value_file1=attrval - ) + values = [attrval, None] else: assert origin_style == "input_local" # Record in source as a variable-local attribute - self.create_roundtrip_testcase( - attr_name=local_attr, vars_values_file1=attrval - ) + values = [None, attrval] + self.run_roundtrip_testcase(attr_name=local_attr, values=values) - if local_attr in iris.fileformats.netcdf.saver._CF_DATA_ATTRS: - # These ones are simply discarded on loading. - # By experiment, this overlap between _CF_ATTRS and _CF_DATA_ATTRS - # currently contains only 'missing_value' and 'standard_error_multiplier'. + if ( + local_attr in ("missing_value", "standard_error_multiplier") + and origin_style == "input_local" + ): + # These ones are actually discarded by roundtrip. + # Not clear why, but for now this captures the facts. expect_global = None expect_var = None else: @@ -556,6 +1007,7 @@ def test_16_localstyle(self, local_attr, origin_style): if ( local_attr == "ukmo__process_flags" and origin_style == "input_global" + and not do_split ): # This is very odd behaviour + surely unintended. # It's supposed to handle vector values (which we are not checking). @@ -564,14 +1016,40 @@ def test_16_localstyle(self, local_attr, origin_style): attrval = "p r o c e s s" expect_var = attrval - if local_attr == "STASH": + if local_attr == "STASH" and ( + origin_style == "input_local" or not do_split + ): # A special case, output translates this to a different attribute name. self.attrname = "um_stash_source" - self.check_roundtrip_results( - global_attr_value=expect_global, - var_attr_vals=expect_var, - ) + expected_result = [expect_global, expect_var] + if do_split and origin_style == "input_global": + # The result is simply the "other way around" + expected_result = expected_result[::-1] + self.check_roundtrip_results(expected_result) + + @pytest.mark.parametrize("testcase", _MATRIX_TESTCASES[:max_param_attrs]) + @pytest.mark.parametrize("attrname", _MATRIX_ATTRNAMES) + def test_roundtrip_matrix( + self, testcase, attrname, matrix_results, do_split + ): + do_saves, matrix_results = matrix_results + split_param = "split" if do_split else "unsplit" + testcase_spec = matrix_results["roundtrip"][testcase] + input_spec = testcase_spec["input"] + values = decode_matrix_input(input_spec) + + self.run_roundtrip_testcase(attrname, values) + results = self.fetch_results(filepath=self.result_filepath) + result_spec = encode_matrix_result(results) + + attr_style = deduce_attr_style(attrname) + expected = testcase_spec[attr_style][split_param] + + if do_saves: + testcase_spec[attr_style][split_param] = result_spec + if expected is not None: + assert result_spec == expected class TestLoad(MixinAttrsTesting): @@ -588,33 +1066,24 @@ class TestLoad(MixinAttrsTesting): """ - def create_load_testcase( - self, - attr_name, - global_value_file1=None, - vars_values_file1=None, - global_value_file2=None, - vars_values_file2=None, - ) -> iris.cube.CubeList: - """ - Initialise the testcase from the passed-in controls, configure the input - files and run a save-load roundtrip to produce the output file. - - The name of the tested attribute and all the temporary filepaths are stored - on the instance, from where "self.check_load_results()" can get them. - - """ - self.attrname = attr_name - self.input_filepaths = self.create_testcase_files( - attr_name=attr_name, - global_value_file1=global_value_file1, - var_values_file1=vars_values_file1, - global_value_file2=global_value_file2, - var_values_file2=vars_values_file2, + def run_load_testcase(self, attr_name, values): + self.run_testcase( + attr_name=attr_name, values=values, create_cubes_or_files="files" ) + + def check_load_results(self, expected, oldstyle_combined=False): + if not _SPLIT_SAVE_SUPPORTED and not oldstyle_combined: + # Don't check "newstyle" in the old world -- just skip it. + return result_cubes = iris.load(self.input_filepaths) - result_cubes = sorted(result_cubes, key=lambda cube: cube.name()) - return result_cubes + results = self.fetch_results( + cubes=result_cubes, oldstyle_combined=oldstyle_combined + ) + # Standardise expected form to list(lists). + assert isinstance(expected, list) + if not isinstance(expected[0], list): + expected = [expected] + assert results == expected ####################################################### # Tests on "user-style" attributes. @@ -623,83 +1092,58 @@ def create_load_testcase( # def test_01_userstyle_single_global(self): - cube1, cube2 = self.create_load_testcase( - attr_name="myname", # A generic "user" attribute with no special handling - global_value_file1="single-value", - vars_values_file1={ - "myvar": None, - "myvar2": None, - }, # the variable has no such attribute + self.run_load_testcase( + attr_name="myname", values=["single_value", None, None] ) - # Default behaviour for a general global user-attribute. - # It is attached to all loaded cubes. - - expected_dict = {"myname": "single-value"} - for cube in (cube1, cube2): - # #1 : legacy results, for cube.attributes **viewed as a plain dictionary**. - assert dict(cube1.attributes) == expected_dict - # #2 : exact expected result, viewed as newstyle split-attributes - assert cube1.attributes == CubeAttrsDict(globals=expected_dict) + # Legacy-equivalent result check (single attributes dict per cube) + self.check_load_results( + [None, "single_value", "single_value"], + oldstyle_combined=True, + ) + # Full new-style results check + self.check_load_results(["single_value", None, None]) def test_02_userstyle_single_local(self): # Default behaviour for a general local user-attribute. # It is attached to only the specific cube. - cube1, cube2 = self.create_load_testcase( + self.run_load_testcase( attr_name="myname", # A generic "user" attribute with no special handling - vars_values_file1={"myvar1": "single-value", "myvar2": None}, + values=[None, "single-value", None], + ) + self.check_load_results( + [None, "single-value", None], oldstyle_combined=True ) - assert cube1.attributes == {"myname": "single-value"} - assert cube2.attributes == {} + self.check_load_results([None, "single-value", None]) def test_03_userstyle_multiple_different(self): # Default behaviour for differing local user-attributes. # The global attribute is simply lost, because there are local ones. - vars1 = {"f1_v1": "f1v1", "f1_v2": "f1v2"} - vars2 = {"f2_v1": "x1", "f2_v2": "x2"} - cube1, cube2, cube3, cube4 = self.create_load_testcase( + self.run_load_testcase( attr_name="random", # A generic "user" attribute with no special handling - global_value_file1="global_file1", - vars_values_file1=vars1, - global_value_file2="global_file2", - vars_values_file2=vars2, - ) - - # (#1) : legacy equivalence : for cube.attributes viewed as a plain 'dict' - assert dict(cube1.attributes) == {"random": "f1v1"} - assert dict(cube2.attributes) == {"random": "f1v2"} - assert dict(cube3.attributes) == {"random": "x1"} - assert dict(cube4.attributes) == {"random": "x2"} - - # (#1) : exact results check, for newstyle "split" cube attrs - assert cube1.attributes == CubeAttrsDict( - globals={"random": "global_file1"}, locals={"random": "f1v1"} - ) - assert cube2.attributes == CubeAttrsDict( - globals={"random": "global_file1"}, locals={"random": "f1v2"} + values=[ + ["global_file1", "f1v1", "f1v2"], + ["global_file2", "x1", "x2"], + ], ) - assert cube3.attributes == CubeAttrsDict( - globals={"random": "global_file2"}, locals={"random": "x1"} + self.check_load_results( + [None, "f1v1", "f1v2", "x1", "x2"], + oldstyle_combined=True, ) - assert cube4.attributes == CubeAttrsDict( - globals={"random": "global_file2"}, locals={"random": "x2"} + self.check_load_results( + [["global_file1", "f1v1", "f1v2"], ["global_file2", "x1", "x2"]] ) def test_04_userstyle_multiple_same(self): # Nothing special to note in this case # TODO: ??remove?? - cube1, cube2 = self.create_load_testcase( + self.run_load_testcase( attr_name="random", - global_value_file1="global_file1", - vars_values_file1={"v1": "same-value", "v2": "same-value"}, - ) - for cube in (cube1, cube2): - # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == {"random": "same-value"} - # (#2): exact results, with newstyle "split" cube attrs - assert cube2.attributes == CubeAttrsDict( - globals={"random": "global_file1"}, - locals={"random": "same-value"}, - ) + values=["global_file1", "same-value", "same-value"], + ) + self.check_load_results( + oldstyle_combined=True, expected=[None, "same-value", "same-value"] + ) + self.check_load_results(["global_file1", "same-value", "same-value"]) ####################################################### # Tests on "Conventions" attribute. @@ -714,28 +1158,27 @@ def test_04_userstyle_multiple_same(self): def test_07_conventions_var_local(self): # What happens if 'Conventions' appears as a variable-local attribute. # N.B. this is not good CF, but we'll see what happens anyway. - (cube,) = self.create_load_testcase( + self.run_load_testcase( attr_name="Conventions", - global_value_file1=None, - vars_values_file1="user_set", + values=[None, "user_set"], ) - assert cube.attributes == {"Conventions": "user_set"} + # Legacy result + self.check_load_results([None, "user_set"], oldstyle_combined=True) + # Newstyle result + self.check_load_results([None, "user_set"]) def test_08_conventions_var_both(self): # What happens if 'Conventions' appears as both global + local attribute. - # = the global version gets lost. - (cube,) = self.create_load_testcase( + self.run_load_testcase( attr_name="Conventions", - global_value_file1="global-setting", - vars_values_file1="local-setting", + values=["global-setting", "local-setting"], ) - # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == {"Conventions": "local-setting"} - # (#2): exact results, with newstyle "split" cube attrs - assert cube.attributes == CubeAttrsDict( - globals={"Conventions": "global-setting"}, - locals={"Conventions": "local-setting"}, + # (#1): legacy result : the global version gets lost. + self.check_load_results( + [None, "local-setting"], oldstyle_combined=True ) + # (#2): newstyle results : retain both. + self.check_load_results(["global-setting", "local-setting"]) ####################################################### # Tests on "global" style attributes @@ -744,74 +1187,67 @@ def test_08_conventions_var_both(self): def test_09_globalstyle__global(self, global_attr): attr_content = f"Global tracked {global_attr}" - (cube,) = self.create_load_testcase( - attr_name=global_attr, - global_value_file1=attr_content, + self.run_load_testcase( + attr_name=global_attr, values=[attr_content, None] ) - assert cube.attributes == {global_attr: attr_content} + # (#1) legacy + self.check_load_results([None, attr_content], oldstyle_combined=True) + # (#2) newstyle : global status preserved. + self.check_load_results([attr_content, None]) def test_10_globalstyle__local(self, global_attr): # Strictly, not correct CF, but let's see what it does with it. - # = treated the same as a global setting attr_content = f"Local tracked {global_attr}" - (cube,) = self.create_load_testcase( + self.run_load_testcase( attr_name=global_attr, - vars_values_file1=attr_content, + values=[None, attr_content], ) - # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == {global_attr: attr_content} - # (#2): exact results, with newstyle "split" cube attrs - assert cube.attributes == CubeAttrsDict( - locals={global_attr: attr_content} + # (#1): legacy result = treated the same as a global setting + self.check_load_results([None, attr_content], oldstyle_combined=True) + # (#2): newstyle result : remains local + self.check_load_results( + [None, attr_content], ) def test_11_globalstyle__both(self, global_attr): attr_global = f"Global-{global_attr}" attr_local = f"Local-{global_attr}" - (cube,) = self.create_load_testcase( + self.run_load_testcase( attr_name=global_attr, - global_value_file1=attr_global, - vars_values_file1=attr_local, - ) - # promoted local setting "wins" - # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == {global_attr: attr_local} - # (#2): exact results, with newstyle "split" cube attrs - assert cube.attributes == CubeAttrsDict( - globals={global_attr: attr_global}, - locals={global_attr: attr_local}, + values=[attr_global, attr_local], ) + # (#1) legacy result : promoted local setting "wins" + self.check_load_results([None, attr_local], oldstyle_combined=True) + # (#2) newstyle result : both retained + self.check_load_results([attr_global, attr_local]) def test_12_globalstyle__multivar_different(self, global_attr): # Multiple *different* local settings are retained attr_1 = f"Local-{global_attr}-1" attr_2 = f"Local-{global_attr}-2" - cube1, cube2 = self.create_load_testcase( + self.run_load_testcase( attr_name=global_attr, - vars_values_file1={"v1": attr_1, "v2": attr_2}, + values=[None, attr_1, attr_2], ) # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube1.attributes) == {global_attr: attr_1} - assert dict(cube2.attributes) == {global_attr: attr_2} + self.check_load_results([None, attr_1, attr_2], oldstyle_combined=True) # (#2): exact results, with newstyle "split" cube attrs - assert cube1.attributes == CubeAttrsDict(locals={global_attr: attr_1}) - assert cube2.attributes == CubeAttrsDict(locals={global_attr: attr_2}) + self.check_load_results([None, attr_1, attr_2]) def test_14_globalstyle__multifile_different(self, global_attr): - # Different global attributes from multiple files are retained as local ones + # Different global attributes from multiple files attr_1 = f"Global-{global_attr}-1" attr_2 = f"Global-{global_attr}-2" - cube1, cube2, cube3, cube4 = self.create_load_testcase( + self.run_load_testcase( attr_name=global_attr, - global_value_file1=attr_1, - vars_values_file1={"f1v1": None, "f1v2": None}, - global_value_file2=attr_2, - vars_values_file2={"f2v1": None, "f2v2": None}, + values=[[attr_1, None, None], [attr_2, None, None]], + ) + # (#1) legacy : multiple globals retained as local ones + self.check_load_results( + [None, attr_1, attr_1, attr_2, attr_2], oldstyle_combined=True ) - assert cube1.attributes == {global_attr: attr_1} - assert cube2.attributes == {global_attr: attr_1} - assert cube3.attributes == {global_attr: attr_2} - assert cube4.attributes == {global_attr: attr_2} + # (#1) newstyle : result same as input + self.check_load_results([[attr_1, None, None], [attr_2, None, None]]) ####################################################### # Tests on "local" style attributes @@ -837,38 +1273,67 @@ def test_16_localstyle(self, local_attr, origin_style): # Create testfiles and load them, which should always produce a single cube. if origin_style == "input_global": # Record in source as a global attribute - (cube,) = self.create_load_testcase( - attr_name=local_attr, global_value_file1=attrval - ) + values = [attrval, None] else: assert origin_style == "input_local" # Record in source as a variable-local attribute - (cube,) = self.create_load_testcase( - attr_name=local_attr, vars_values_file1=attrval - ) + values = [None, attrval] + + self.run_load_testcase(attr_name=local_attr, values=values) # Work out the expected result. - # NOTE: generally, result will be the same whether the original attribute is - # provided as a global or variable attribute ... - expected_result = {local_attr: attrval} - # ... but there are some special cases + result_value = attrval + # ... there are some special cases if origin_style == "input_local": if local_attr == "ukmo__process_flags": # Some odd special behaviour here. - expected_result = {local_attr: ("process",)} + result_value = (result_value,) elif local_attr in ("standard_error_multiplier", "missing_value"): # For some reason, these ones never appear on the cube - expected_result = {} + result_value = None + + # NOTE: **legacy** result is the same, whether the original attribute was + # provided as a global or local attribute ... + expected_result_legacy = [None, result_value] + # While 'newstyle' results preserve the input type local/global. if origin_style == "input_local": - expected_result_newstyle = CubeAttrsDict(expected_result) + expected_result_newstyle = [None, result_value] else: - expected_result_newstyle = CubeAttrsDict(globals=expected_result) + expected_result_newstyle = [result_value, None] # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == expected_result + self.check_load_results(expected_result_legacy, oldstyle_combined=True) # (#2): exact results, with newstyle "split" cube attrs - assert cube.attributes == expected_result_newstyle + self.check_load_results(expected_result_newstyle) + + @pytest.mark.parametrize("testcase", _MATRIX_TESTCASES[:max_param_attrs]) + @pytest.mark.parametrize("attrname", _MATRIX_ATTRNAMES) + @pytest.mark.parametrize("resultstyle", _MATRIX_LOAD_RESULTSTYLES) + def test_load_matrix( + self, testcase, attrname, matrix_results, resultstyle + ): + do_saves, matrix_results = matrix_results + testcase_spec = matrix_results["load"][testcase] + input_spec = testcase_spec["input"] + values = decode_matrix_input(input_spec) + + self.run_load_testcase(attrname, values) + + result_cubes = iris.load(self.input_filepaths) + do_combined = resultstyle == "legacy" + results = self.fetch_results( + cubes=result_cubes, oldstyle_combined=do_combined + ) + result_spec = encode_matrix_result(results) + + attr_style = deduce_attr_style(attrname) + expected = testcase_spec[attr_style][resultstyle] + + if do_saves: + testcase_spec[attr_style][resultstyle] = result_spec + if expected is not None: + assert result_spec == expected class TestSave(MixinAttrsTesting): @@ -877,120 +1342,184 @@ class TestSave(MixinAttrsTesting): """ - def create_save_testcase(self, attr_name, value1, value2=None): + # Parametrise all tests over split/unsplit saving. + @pytest.fixture( + params=_SPLIT_PARAM_VALUES, ids=_SPLIT_PARAM_IDS, autouse=True + ) + def do_split(self, request): + do_split = request.param + self.save_split_attrs = do_split + return do_split + + def run_save_testcase(self, attr_name: str, values: list): + # Create input cubes. + self.run_testcase( + attr_name=attr_name, + values=values, + create_cubes_or_files="cubes", + ) + + # Save input cubes to a temporary result file. + with warnings.catch_warnings(record=True) as captured_warnings: + self.result_filepath = self._testfile_path("result") + do_split = getattr(self, "save_split_attrs", False) + kwargs = ( + dict(save_split_attrs=do_split) + if _SPLIT_SAVE_SUPPORTED + else dict() + ) + with iris.FUTURE.context(**kwargs): + iris.save(self.input_cubes, self.result_filepath) + + self.captured_warnings = captured_warnings + + def run_save_testcase_legacytype(self, attr_name: str, values: list): """ - Test attribute saving for cube(s) with given value(s). + Legacy-type means : before cubes had split attributes. - Create cubes(s) and save to temporary file, then return the global and all - variable-local attributes of that name (or None-s) from the file. + This just means we have only one "set" of cubes, with ***no*** distinct global + attribute. """ - self.attrname = ( - attr_name # Required for common testfile-naming function. - ) - if value2 is None: - n_cubes = 1 - values = [value1] - else: - n_cubes = 2 - values = [value1, value2] - cube_names = [f"cube_{i_cube}" for i_cube in range(n_cubes)] - cubes = [ - Cube([0], long_name=cube_name, attributes={attr_name: attr_value}) - for cube_name, attr_value in zip(cube_names, values) - ] - self.result_filepath = self._testfile_path("result") - iris.save(cubes, self.result_filepath) - # Get the global+local attribute values directly from the file with netCDF4 - if attr_name == "STASH": - # A special case : the stored name is different - attr_name = "um_stash_source" - try: - ds = threadsafe_nc4.DatasetWrapper(self.result_filepath) - global_result = ( - ds.getncattr(attr_name) if attr_name in ds.ncattrs() else None - ) - local_results = [ - ( - var.getncattr(attr_name) - if attr_name in var.ncattrs() - else None - ) - for var in ds.variables.values() - ] - finally: - ds.close() - return [global_result] + local_results + if not isinstance(values, list): + # Translate single input value to list-of-1 + values = [values] - def test_01_userstyle__single(self): - results = self.create_save_testcase("random", "value-x") - # It is stored as a *global* by default. - assert results == ["value-x", None] + self.run_save_testcase(attr_name, [None] + values) - def test_02_userstyle__multiple_same(self): - results = self.create_save_testcase("random", "value-x", "value-x") - # As above. - assert results == ["value-x", None, None] + def check_save_results( + self, expected: list, expected_warnings: List[str] = None + ): + results = self.fetch_results(filepath=self.result_filepath) + assert results == expected + check_captured_warnings(expected_warnings, self.captured_warnings) + + def test_userstyle__single(self, do_split): + self.run_save_testcase_legacytype("random", "value-x") + if do_split: + # result as input values + expected_result = [None, "value-x"] + else: + # in legacy mode, promoted = stored as a *global* by default. + expected_result = ["value-x", None] + self.check_save_results(expected_result) + + def test_userstyle__multiple_same(self, do_split): + self.run_save_testcase_legacytype("random", ["value-x", "value-x"]) + if do_split: + # result as input values + expected_result = [None, "value-x", "value-x"] + else: + # in legacy mode, promoted = stored as a *global* by default. + expected_result = ["value-x", None, None] + self.check_save_results(expected_result) - def test_03_userstyle__multiple_different(self): - results = self.create_save_testcase("random", "value-A", "value-B") + def test_userstyle__multiple_different(self): # Clashing values are stored as locals on the individual variables. - assert results == [None, "value-A", "value-B"] + self.run_save_testcase_legacytype("random", ["value-A", "value-B"]) + self.check_save_results([None, "value-A", "value-B"]) + + def test_userstyle__multiple_onemissing(self): + # Multiple user-type, with one missing, behave like different values. + self.run_save_testcase_legacytype( + "random", + ["value", None], + ) + # Stored as locals when there are differing values. + self.check_save_results([None, "value", None]) - def test_04_Conventions__single(self): - results = self.create_save_testcase("Conventions", "x") + def test_Conventions__single(self): + self.run_save_testcase_legacytype("Conventions", "x") # Always discarded + replaced by a single global setting. - assert results == ["CF-1.7", None] + self.check_save_results(["CF-1.7", None]) - def test_05_Conventions__multiple_same(self): - results = self.create_save_testcase( - "Conventions", "same-value", "same-value" + def test_Conventions__multiple_same(self): + self.run_save_testcase_legacytype( + "Conventions", ["same-value", "same-value"] ) # Always discarded + replaced by a single global setting. - assert results == ["CF-1.7", None, None] + self.check_save_results(["CF-1.7", None, None]) - def test_06_Conventions__multiple_different(self): - results = self.create_save_testcase( - "Conventions", "value-A", "value-B" + def test_Conventions__multiple_different(self): + self.run_save_testcase_legacytype( + "Conventions", ["value-A", "value-B"] ) # Always discarded + replaced by a single global setting. - assert results == ["CF-1.7", None, None] - - def test_07_globalstyle__single(self, global_attr): - results = self.create_save_testcase(global_attr, "value") - # Defaults to global - assert results == ["value", None] - - def test_08_globalstyle__multiple_same(self, global_attr): - results = self.create_save_testcase( - global_attr, "value-same", "value-same" + self.check_save_results(["CF-1.7", None, None]) + + def test_globalstyle__single(self, global_attr, do_split): + self.run_save_testcase_legacytype(global_attr, ["value"]) + if do_split: + # result as input values + expected_warning = "should only be a CF global" + expected_result = [None, "value"] + else: + # in legacy mode, promoted + expected_warning = None + expected_result = ["value", None] + self.check_save_results(expected_result, expected_warning) + + def test_globalstyle__multiple_same(self, global_attr, do_split): + # Multiple global-type with same values are made global. + self.run_save_testcase_legacytype( + global_attr, + ["value-same", "value-same"], + ) + if do_split: + # result as input values + expected_result = [None, "value-same", "value-same"] + expected_warning = "should only be a CF global attribute" + else: + # in legacy mode, promoted + expected_result = ["value-same", None, None] + expected_warning = None + self.check_save_results(expected_result, expected_warning) + + def test_globalstyle__multiple_different(self, global_attr): + # Multiple global-type with different values become local, with warning. + self.run_save_testcase_legacytype(global_attr, ["value-A", "value-B"]) + # *Only* stored as locals when there are differing values. + msg_regexp = ( + f"'{global_attr}' is being added as CF data variable attribute," + f".* should only be a CF global attribute." + ) + self.check_save_results( + [None, "value-A", "value-B"], expected_warnings=msg_regexp ) - assert results == ["value-same", None, None] - def test_09_globalstyle__multiple_different(self, global_attr): + def test_globalstyle__multiple_onemissing(self, global_attr): + # Multiple global-type, with one missing, behave like different values. + self.run_save_testcase_legacytype( + global_attr, ["value", "value", None] + ) + # Stored as locals when there are differing values. msg_regexp = ( f"'{global_attr}' is being added as CF data variable attribute," f".* should only be a CF global attribute." ) - with pytest.warns(UserWarning, match=msg_regexp): - results = self.create_save_testcase( - global_attr, "value-A", "value-B" - ) - # *Only* stored as locals when there are differing values. - assert results == [None, "value-A", "value-B"] + self.check_save_results( + [None, "value", "value", None], expected_warnings=msg_regexp + ) + + def test_localstyle__single(self, local_attr): + self.run_save_testcase_legacytype(local_attr, ["value"]) - def test_10_localstyle__single(self, local_attr): - results = self.create_save_testcase(local_attr, "value") # Defaults to local expected_results = [None, "value"] + # .. but a couple of special cases if local_attr == "ukmo__process_flags": # A particular, really weird case expected_results = [None, "v a l u e"] - assert results == expected_results + elif local_attr == "STASH": + # A special case : the stored name is different + self.attrname = "um_stash_source" + + self.check_save_results(expected_results) - def test_11_localstyle__multiple_same(self, local_attr): - results = self.create_save_testcase( - local_attr, "value-same", "value-same" + def test_localstyle__multiple_same(self, local_attr): + self.run_save_testcase_legacytype( + local_attr, ["value-same", "value-same"] ) + # They remain separate + local expected_results = [None, "value-same", "value-same"] if local_attr == "ukmo__process_flags": @@ -1000,10 +1529,14 @@ def test_11_localstyle__multiple_same(self, local_attr): "v a l u e - s a m e", "v a l u e - s a m e", ] - assert results == expected_results + elif local_attr == "STASH": + # A special case : the stored name is different + self.attrname = "um_stash_source" + + self.check_save_results(expected_results) - def test_12_localstyle__multiple_different(self, local_attr): - results = self.create_save_testcase(local_attr, "value-A", "value-B") + def test_localstyle__multiple_different(self, local_attr): + self.run_save_testcase_legacytype(local_attr, ["value-A", "value-B"]) # Different values are treated just the same as matching ones. expected_results = [None, "value-A", "value-B"] if local_attr == "ukmo__process_flags": @@ -1013,4 +1546,103 @@ def test_12_localstyle__multiple_different(self, local_attr): "v a l u e - A", "v a l u e - B", ] - assert results == expected_results + elif local_attr == "STASH": + # A special case : the stored name is different + self.attrname = "um_stash_source" + self.check_save_results(expected_results) + + # + # Test handling of newstyle independent global+local cube attributes. + # + def test_globallocal_clashing(self, do_split): + # A cube has clashing local + global attrs. + original_values = ["valueA", "valueB"] + self.run_save_testcase("userattr", original_values) + expected_result = original_values.copy() + if not do_split: + # in legacy mode, "promote" = lose the local one + expected_result[0] = expected_result[1] + expected_result[1] = None + self.check_save_results(expected_result) + + def test_globallocal_oneeach_same(self, do_split): + # One cube with global attr, another with identical local one. + self.run_save_testcase( + "userattr", values=[[None, "value"], ["value", None]] + ) + if do_split: + expected = [None, "value", "value"] + expected_warning = ( + r"Saving the cube global attributes \['userattr'\] as local" + ) + else: + # N.B. legacy code sees only two equal values (and promotes). + expected = ["value", None, None] + expected_warning = None + + self.check_save_results(expected, expected_warning) + + def test_globallocal_oneeach_different(self, do_split): + # One cube with global attr, another with a *different* local one. + self.run_save_testcase( + "userattr", [[None, "valueA"], ["valueB", None]] + ) + if do_split: + warning = ( + r"Saving the cube global attributes \['userattr'\] as local" + ) + else: + # N.B. legacy code does not warn of global-to-local "demotion". + warning = None + self.check_save_results([None, "valueA", "valueB"], warning) + + def test_globallocal_one_other_clashingglobals(self, do_split): + # Two cubes with both, second cube has a clashing global attribute. + self.run_save_testcase( + "userattr", + values=[["valueA", "valueB"], ["valueXXX", "valueB"]], + ) + if do_split: + expected = [None, "valueB", "valueB"] + expected_warnings = [ + "Saving.* global attributes.* as local", + 'attributes.* of cube "v1" were not saved', + 'attributes.* of cube "v2" were not saved', + ] + else: + # N.B. legacy code sees only the locals, and promotes them. + expected = ["valueB", None, None] + expected_warnings = None + self.check_save_results(expected, expected_warnings) + + def test_globallocal_one_other_clashinglocals(self, do_split): + # Two cubes with both, second cube has a clashing local attribute. + inputs = [["valueA", "valueB"], ["valueA", "valueXXX"]] + if do_split: + expected = ["valueA", "valueB", "valueXXX"] + else: + # N.B. legacy code sees only the locals. + expected = [None, "valueB", "valueXXX"] + self.run_save_testcase("userattr", values=inputs) + self.check_save_results(expected) + + @pytest.mark.parametrize("testcase", _MATRIX_TESTCASES[:max_param_attrs]) + @pytest.mark.parametrize("attrname", _MATRIX_ATTRNAMES) + def test_save_matrix(self, testcase, attrname, matrix_results, do_split): + do_saves, matrix_results = matrix_results + split_param = "split" if do_split else "unsplit" + testcase_spec = matrix_results["save"][testcase] + input_spec = testcase_spec["input"] + values = decode_matrix_input(input_spec) + + self.run_save_testcase(attrname, values) + results = self.fetch_results(filepath=self.result_filepath) + result_spec = encode_matrix_result(results) + + attr_style = deduce_attr_style(attrname) + expected = testcase_spec[attr_style][split_param] + + if do_saves: + testcase_spec[attr_style][split_param] = result_spec + if expected is not None: + assert result_spec == expected