From aa0081a7a01ec6667090df671671612393b0d2be Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Jul 2024 19:29:08 +0200 Subject: [PATCH 1/5] extra metadata dict, initialized inside from_owl_uri --- cdci_data_analysis/analysis/parameters.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index 03364984..4dc56c5f 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -210,6 +210,8 @@ def __init__(self, allowed_values=None, min_value = None, max_value = None, + + extra_metadata=None, **kwargs ): @@ -261,6 +263,7 @@ def __init__(self, self._max_value = max_value self._bound_units = self.units self.value = value + self.extra_metadata = extra_metadata self._arg_list = [self.name] @@ -463,6 +466,8 @@ def reprJSONifiable(self): restrictions['schema'] = self.schema if restrictions: reprjson[0]['restrictions'] = restrictions + if getattr(self, 'extra_metadata', None) is not None: + reprjson[0]['extra_metadata'] = self.extra_metadata if getattr(self, 'owl_uris', None): if isinstance(self.owl_uris, str): reprjson[0]['owl_uri'] = [ self.owl_uris ] @@ -514,13 +519,16 @@ def from_owl_uri(cls, par_unit = onto.get_parameter_unit(owl_uri) min_value, max_value = onto.get_limits(owl_uri) allowed_values = onto.get_allowed_values(owl_uri) - + label = onto.get_oda_label(owl_uri) + for owl_superclass_uri in parameter_hierarchy: for python_subclass in subclasses_recursive(cls): logger.debug("searching for class with owl_uri=%s, found %s", owl_superclass_uri, python_subclass) if python_subclass.matches_owl_uri(owl_superclass_uri): logger.info("will construct %s by owl_uri %s", python_subclass, owl_superclass_uri) - call_kwargs = {} + call_kwargs = {'extra_metadata': {}} + if label is not None: + call_kwargs['extra_metadata']['label'] = label call_signature = signature(python_subclass) var_kw_signature = call_parameter.VAR_KEYWORD in [x.kind for x in call_signature.parameters.values()] @@ -566,7 +574,7 @@ def from_owl_uri(cls, class String(Parameter): owl_uris = ("http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String") - def __init__(self, value=None, name_format='str', name=None, allowed_values = None): + def __init__(self, value=None, name_format='str', name=None, allowed_values = None, extra_metadata = None): _allowed_units = ['str'] super().__init__(value=value, @@ -574,7 +582,8 @@ def __init__(self, value=None, name_format='str', name=None, allowed_values = No check_value=self.check_name_value, name=name, allowed_units=_allowed_units, - allowed_values=allowed_values) + allowed_values=allowed_values, + extra_metadata=extra_metadata) @staticmethod def check_name_value(value, units=None, name=None, par_format=None): From fc84c2f4ac1d5e32f0d12ec797a6cbceac4bc60a Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Jul 2024 20:08:34 +0200 Subject: [PATCH 2/5] extra_metadata args for more Parameter classes --- cdci_data_analysis/analysis/parameters.py | 77 ++++++++++++++--------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index 4dc56c5f..bdb478b2 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -499,7 +499,7 @@ def from_owl_uri(cls, 'Trying to find parameter which have %s directly set. ' 'extra_ttl will be ignored ', owl_uri) parameter_hierarchy = [ owl_uri ] - par_format = par_unit = allowed_values = min_value = max_value = None + par_format = par_unit = allowed_values = min_value = max_value = label = None else: if ontology_path is not None: if isinstance(ontology_path, (str, os.PathLike)): @@ -680,7 +680,8 @@ def __init__(self, check_value=None, min_value= None, max_value = None, - units_name = None): + units_name = None, + extra_metadata=None): super().__init__(value=value, units=units, @@ -693,7 +694,8 @@ def __init__(self, allowed_units=allowed_units, min_value=min_value, max_value=max_value, - units_name = units_name) + units_name = units_name, + extra_metadata=extra_metadata) @@ -709,7 +711,8 @@ def __init__(self, max_value = None, units_name = None, default_units = None, - allowed_units = None): + allowed_units = None, + extra_metadata=None): super().__init__(value=value, units=units, @@ -721,7 +724,8 @@ def __init__(self, allowed_units=allowed_units, min_value = min_value, max_value = max_value, - units_name = units_name) + units_name = units_name, + extra_metadata=extra_metadata) def set_par_internal_value(self, value): if isinstance(value, float): @@ -734,13 +738,14 @@ class Time(Parameter): owl_uris = ("http://odahub.io/ontology#TimeInstant",) format_kw = 'T_format' - def __init__(self, value=None, T_format='isot', name=None, Time_format_name='T_format', par_default_format='isot'): + def __init__(self, value=None, T_format='isot', name=None, Time_format_name='T_format', par_default_format='isot', extra_metadata=None): super().__init__(value=value, par_format=T_format, par_format_name=Time_format_name, par_default_format=par_default_format, - name=name) + name=name, + extra_metadata=extra_metadata) def get_default_value(self): return self.get_value_in_default_format() @@ -785,7 +790,7 @@ class TimeDelta(Time): owl_uris = ("http://odahub.io/ontology#TimeDeltaIsDeprecated",) format_kw = 'delta_T_format' - def __init__(self, value=None, delta_T_format='sec', name=None, delta_T_format_name=None, par_default_format='sec'): + def __init__(self, value=None, delta_T_format='sec', name=None, delta_T_format_name=None, par_default_format='sec', extra_metadata=None): logging.warning(('TimeDelta parameter is deprecated. ' 'It derives from Time, which is confusing. ' 'Consider using TimeInterval parameter.')) @@ -793,7 +798,8 @@ def __init__(self, value=None, delta_T_format='sec', name=None, delta_T_format_n T_format=delta_T_format, Time_format_name=delta_T_format_name, par_default_format=par_default_format, - name=name) + name=name, + extra_metadata=extra_metadata) def get_value_in_format(self, units): return getattr(self._astropy_time_delta, units) @@ -825,7 +831,8 @@ def __init__(self, default_units='s', min_value=None, max_value=None, - units_name = None): + units_name = None, + extra_metadata=None): _allowed_units = ['s', 'minute', 'hour', 'day', 'year'] super().__init__(value=value, @@ -835,13 +842,14 @@ def __init__(self, min_value=min_value, max_value=max_value, units_name = units_name, - allowed_units=_allowed_units) + allowed_units=_allowed_units, + extra_metadata=extra_metadata) class InputProdList(Parameter): owl_uris = ('http://odahub.io/ontology#InputProdList',) # TODO removal of the leading underscore cannot be done for compatibility with the plugins - def __init__(self, value=None, _format='names_list', name: str = None): + def __init__(self, value=None, _format='names_list', name: str = None, extra_metadata=None): _allowed_units = ['names_list'] if value is None: @@ -851,7 +859,8 @@ def __init__(self, value=None, _format='names_list', name: str = None): par_format=_format, check_value=self.check_list_value, name=name, - allowed_units=_allowed_units) + allowed_units=_allowed_units, + extra_metadata=extra_metadata) @staticmethod def _split(str_list): @@ -912,7 +921,8 @@ def __init__(self, name=None, min_value = None, max_value = None, - units_name = None): + units_name = None, + extra_metadata=None): super().__init__(value=value, units=units, @@ -922,7 +932,8 @@ def __init__(self, allowed_units=None, min_value = min_value, max_value = max_value, - units_name = units_name) + units_name = units_name, + extra_metadata=extra_metadata) class Energy(Float): @@ -937,7 +948,8 @@ def __init__(self, check_value=None, min_value = None, max_value = None, - units_name = None): + units_name = None, + extra_metadata=None): _allowed_units = ['keV', 'eV', 'MeV', 'GeV', 'TeV', 'Hz', 'MHz', 'GHz'] @@ -949,7 +961,8 @@ def __init__(self, allowed_units=_allowed_units, min_value = min_value, max_value = max_value, - units_name = units_name) + units_name = units_name, + extra_metadata=extra_metadata) class SpectralBoundary(Energy): def __init__(self, @@ -960,7 +973,8 @@ def __init__(self, check_value=None, min_value=None, max_value=None, - units_name=None): + units_name=None, + extra_metadata=None): # retro-compatibility with integral plugin if check_value is None: @@ -973,7 +987,8 @@ def __init__(self, check_value=check_value, min_value=min_value, max_value=max_value, - units_name=units_name) + units_name=units_name, + extra_metadata=extra_metadata) @staticmethod def check_energy_value(value, units, name): @@ -982,7 +997,7 @@ def check_energy_value(value, units, name): class DetectionThreshold(Float): owl_uris = ("http://odahub.io/ontology#DetectionThreshold",) - def __init__(self, value=None, units='sigma', name=None, min_value = None, max_value = None): + def __init__(self, value=None, units='sigma', name=None, min_value = None, max_value = None, extra_metadata=None): _allowed_units = ['sigma'] super().__init__(value=value, @@ -991,7 +1006,8 @@ def __init__(self, value=None, units='sigma', name=None, min_value = None, max_v name=name, allowed_units=_allowed_units, min_value = min_value, - max_value = max_value) + max_value = max_value, + extra_metadata=extra_metadata) # 'sigma' is not astropy unit, so need to override methods def get_value_in_units(self, units): @@ -1004,24 +1020,26 @@ def set_par_internal_value(self, value): self._value = None class UserCatalog(Parameter): - def __init__(self, value=None, name_format='str', name=None): + def __init__(self, value=None, name_format='str', name=None, extra_metadata=None): _allowed_units = ['str'] super().__init__(value=value, par_format=name_format, check_value=None, name=name, - allowed_units=_allowed_units) + allowed_units=_allowed_units, + extra_metadata=extra_metadata) class Boolean(Parameter): owl_uris = ('http://www.w3.org/2001/XMLSchema#bool',"http://odahub.io/ontology#Boolean") - def __init__(self, value=None, name=None): + def __init__(self, value=None, name=None, extra_metadata=None): self._true_rep = ['True', 'true', 'yes', '1', True] self._false_rep = ['False', 'false', 'no', '0', False] super().__init__(value=value, name=name, - allowed_values=self._true_rep+self._false_rep + allowed_values=self._true_rep+self._false_rep, + extra_metadata = extra_metadata ) @property @@ -1040,7 +1058,7 @@ def value(self, v): class StructuredParameter(Parameter): owl_uris = ("http://odahub.io/ontology#StructuredParameter") - def __init__(self, value=None, name=None, schema={"oneOf": [{"type": "object"}, {"type": "array"}]}): + def __init__(self, value=None, name=None, schema={"oneOf": [{"type": "object"}, {"type": "array"}]}, extra_metadata=None): self.schema = schema @@ -1048,7 +1066,8 @@ def __init__(self, value=None, name=None, schema={"oneOf": [{"type": "object"}, logger.warning("Parameter %s: Schema is not defined, will allow any structure.", name) super().__init__(value=value, - name=name) + name=name, + extra_metadata=extra_metadata) def check_schema(self): if self.schema is not None: @@ -1074,7 +1093,7 @@ def get_default_value(self): class PhosphorosFiltersTable(StructuredParameter): owl_uris = ('http://odahub.io/ontology#PhosphorosFiltersTable') - def __init__(self, value=None, name=None): + def __init__(self, value=None, name=None, extra_metadata=None): # TODO: either list or the whole schema may be loaded from the external file, purely based on URI. # If there is no additional check, this would allow to avoid even having the class. @@ -1111,7 +1130,7 @@ def __init__(self, value=None, name=None): "required": ["filter", "flux", "flux_error"] } - super().__init__(value=value, name=name, schema=schema) + super().__init__(value=value, name=name, schema=schema, extra_metadata=extra_metadata) def additional_check(self): assert len(self._value['filter']) == len(self._value['flux']) == len(self._value['flux_error']) From a7bd523b2682c169441e1655640abdd629f34706 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Jul 2024 20:08:43 +0200 Subject: [PATCH 3/5] path ontology test --- tests/test_parameters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_parameters.py b/tests/test_parameters.py index 3eb47853..779e5beb 100644 --- a/tests/test_parameters.py +++ b/tests/test_parameters.py @@ -1,4 +1,5 @@ import pytest +import os from cdci_data_analysis.analysis.instrument import Instrument from cdci_data_analysis.analysis.queries import ( @@ -422,7 +423,7 @@ def test_boolean_parameter(inval, iswrong, expected): True, 100, Energy), ]) def test_parameter_class_from_owl_uri(uri, extra_ttl, use_ontology, value, param_type): - ontology_path = 'tests/oda-ontology.ttl' if use_ontology else None + ontology_path = os.path.join(os.path.dirname(__file__), 'oda-ontology.ttl') if use_ontology else None param = Parameter.from_owl_uri(uri, value=value, extra_ttl = extra_ttl, From 2a2c184b9eddf2dcd74caf9c8fe1c19ca32aab85 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 16 Jul 2024 15:15:37 +0200 Subject: [PATCH 4/5] code optimization --- cdci_data_analysis/analysis/parameters.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index bdb478b2..6b057d7e 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -492,6 +492,8 @@ def from_owl_uri(cls, **kwargs): from oda_api.ontology_helper import Ontology + metadata_keys = ['label', 'description'] + if ontology_path is not None and ontology_object is not None: raise RuntimeError("Both ontology_path and ontology_object parameters are set.") elif ontology_path is None and ontology_object is None: @@ -499,7 +501,7 @@ def from_owl_uri(cls, 'Trying to find parameter which have %s directly set. ' 'extra_ttl will be ignored ', owl_uri) parameter_hierarchy = [ owl_uri ] - par_format = par_unit = allowed_values = min_value = max_value = label = None + par_format = par_unit = allowed_values = min_value = max_value = label = description = None else: if ontology_path is not None: if isinstance(ontology_path, (str, os.PathLike)): @@ -519,16 +521,17 @@ def from_owl_uri(cls, par_unit = onto.get_parameter_unit(owl_uri) min_value, max_value = onto.get_limits(owl_uri) allowed_values = onto.get_allowed_values(owl_uri) - label = onto.get_oda_label(owl_uri) + label = onto.get_direct_annotation(owl_uri, "label") + description = onto.get_direct_annotation(owl_uri, "description") for owl_superclass_uri in parameter_hierarchy: for python_subclass in subclasses_recursive(cls): logger.debug("searching for class with owl_uri=%s, found %s", owl_superclass_uri, python_subclass) if python_subclass.matches_owl_uri(owl_superclass_uri): logger.info("will construct %s by owl_uri %s", python_subclass, owl_superclass_uri) - call_kwargs = {'extra_metadata': {}} - if label is not None: - call_kwargs['extra_metadata']['label'] = label + call_kwargs = { + 'extra_metadata': {key: val for key, val in zip(metadata_keys, [label, description]) if + val is not None}} call_signature = signature(python_subclass) var_kw_signature = call_parameter.VAR_KEYWORD in [x.kind for x in call_signature.parameters.values()] From 321094265904a67fb5e25b64be47cbf9ce7024ba Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 12 Aug 2024 15:23:08 +0200 Subject: [PATCH 5/5] group extra_metadata --- cdci_data_analysis/analysis/parameters.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index 6b057d7e..ad84535f 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -492,7 +492,7 @@ def from_owl_uri(cls, **kwargs): from oda_api.ontology_helper import Ontology - metadata_keys = ['label', 'description'] + metadata_keys = ['label', 'description', 'group'] if ontology_path is not None and ontology_object is not None: raise RuntimeError("Both ontology_path and ontology_object parameters are set.") @@ -501,7 +501,7 @@ def from_owl_uri(cls, 'Trying to find parameter which have %s directly set. ' 'extra_ttl will be ignored ', owl_uri) parameter_hierarchy = [ owl_uri ] - par_format = par_unit = allowed_values = min_value = max_value = label = description = None + par_format = par_unit = allowed_values = min_value = max_value = label = description = group = None else: if ontology_path is not None: if isinstance(ontology_path, (str, os.PathLike)): @@ -523,6 +523,7 @@ def from_owl_uri(cls, allowed_values = onto.get_allowed_values(owl_uri) label = onto.get_direct_annotation(owl_uri, "label") description = onto.get_direct_annotation(owl_uri, "description") + group = onto.get_direct_annotation(owl_uri, "group") for owl_superclass_uri in parameter_hierarchy: for python_subclass in subclasses_recursive(cls): @@ -530,7 +531,7 @@ def from_owl_uri(cls, if python_subclass.matches_owl_uri(owl_superclass_uri): logger.info("will construct %s by owl_uri %s", python_subclass, owl_superclass_uri) call_kwargs = { - 'extra_metadata': {key: val for key, val in zip(metadata_keys, [label, description]) if + 'extra_metadata': {key: val for key, val in zip(metadata_keys, [label, description, group]) if val is not None}} call_signature = signature(python_subclass) var_kw_signature = call_parameter.VAR_KEYWORD in [x.kind for x in call_signature.parameters.values()]