From e911b6f95e02308326a358c0f863ef58e24fb705 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Fri, 26 Feb 2021 08:08:08 -0800 Subject: [PATCH] v2 (#4) Co-authored-by: Eric Charles --- python/cfgmdl/__init__.py | 4 +- python/cfgmdl/array.py | 2 +- python/cfgmdl/configurable.py | 78 +++++++++++++++-------------------- python/cfgmdl/derived.py | 56 +++++++++++++++++++++---- python/cfgmdl/function.py | 2 + python/cfgmdl/param_holder.py | 7 +++- python/cfgmdl/parameter.py | 2 +- python/cfgmdl/property.py | 21 +++++++--- python/cfgmdl/ref.py | 78 +++++++++++++++++++++++++++++++++++ python/cfgmdl/tools.py | 73 ++++++++++++++++++++++++++++++++ python/cfgmdl/unit.py | 4 ++ python/cfgmdl/utils.py | 65 ++++++++++++++++++++++------- tests/test_derived.py | 20 ++++----- tests/test_model.py | 6 +-- tests/test_property.py | 2 +- tests/test_units.py | 4 +- 16 files changed, 331 insertions(+), 93 deletions(-) create mode 100644 python/cfgmdl/ref.py create mode 100644 python/cfgmdl/tools.py diff --git a/python/cfgmdl/__init__.py b/python/cfgmdl/__init__.py index a564768..87ad055 100644 --- a/python/cfgmdl/__init__.py +++ b/python/cfgmdl/__init__.py @@ -5,11 +5,13 @@ del get_git_version from .unit import Unit +from .ref import Ref from .array import Array from .property import Property -from .derived import Derived +from .derived import Derived, cached from .configurable import Configurable from .choice import Choice from .parameter import Parameter +from .param_holder import ParamHolder from .model import Model from .function import Function diff --git a/python/cfgmdl/array.py b/python/cfgmdl/array.py index 8b55d68..6c1315f 100644 --- a/python/cfgmdl/array.py +++ b/python/cfgmdl/array.py @@ -18,6 +18,6 @@ class Array(Property): def __init__(self, **kwargs): super(Array, self).__init__(**kwargs) - def _cast_type(self, value): + def _cast_type(self, value, obj=None): """Hook took override type casting""" return np.array(value).astype(self.dtype) diff --git a/python/cfgmdl/configurable.py b/python/cfgmdl/configurable.py index 75ee2c9..3d55690 100644 --- a/python/cfgmdl/configurable.py +++ b/python/cfgmdl/configurable.py @@ -1,21 +1,13 @@ #!/usr/bin/env python """ -Classes used to describe aspect of Models. - -The base class is `Property` which describes any one property of a model, -such as the name, or some other fixed property. - -The `Parameter` class describes variable model parameters. - -The `Derived` class describes model properies that are derived -from other model properties. - +Base class for configurable objects """ from collections import OrderedDict as odict from collections.abc import Mapping from .property import Property from .derived import Derived +from .ref import Ref from .utils import Meta, model_docstring @@ -74,9 +66,8 @@ class MyPair(Model): def __init__(self, **kwargs): """ C'tor. Build from a set of keyword arguments. """ - self._properties = self.find_properties() + self._timestamp = 0 self._init_properties(**kwargs) - self._cache() @classmethod def find_properties(cls): @@ -89,19 +80,36 @@ def find_properties(cls): props[key] = val return props + @classmethod + def getp(cls, name): + """Get a particular property of this model""" + props = cls.find_properties() + return props[name] + + @property + def _properties(self): + """Return the properties of this class""" + return self.find_properties() + def __str__(self, indent=0): """ Cast model as a formatted string """ + props = self._properties try: ret = '{0:>{2}}{1}'.format('', self.name, indent) except AttributeError: ret = "%s" % (type(self)) - if not self._properties: #pragma: no cover + if not props: #pragma: no cover return ret ret += '\n{0:>{2}}{1}'.format('', 'Parameters:', indent + 2) - width = len(max(self._properties.keys(), key=len)) - for name in self._properties.keys(): - value = getattr(self, name) + + width = len(max(props.keys(), key=len)) + for prop in props.values(): + name = prop.public_name + if isinstance(prop, Derived): + value = getattr(self, prop.private_name) + else: + value = getattr(self, name) par = '{0!s:{width}} : {1!r}'.format(name, value, width=width) ret += '\n{0:>{2}}{1}'.format('', par, indent + 4) return ret @@ -121,13 +129,16 @@ def update(self, *args, **kwargs): raise ValueError("Argument passed to Model.upate() %s" % args) kwargs = dict(kwargs) + props = self._properties for name, value in kwargs.items(): # Raise KeyError if Property not found try: - prop = self._properties[name] + prop = props[name] except KeyError as err: raise KeyError("Warning: %s does not have properties %s" % (type(self), name)) from err + if isinstance(prop, (Ref, Derived)): + continue attr = getattr(self, '_%s' % name) if isinstance(attr, Configurable): # Set attributes @@ -146,20 +157,17 @@ def _init_properties(self, **kwargs): """ missing = {} kwcopy = kwargs.copy() - for k, p in self._properties.items(): + props = self._properties + for k, p in props.items(): if k not in kwcopy and p.required: missing[k] = p pval = kwcopy.pop(k, p.default) - + if isinstance(p, Ref): + setattr(self, p.owner_name, p.owner) + setattr(self, p.private_name, None) + continue p.__set__(self, pval) - if isinstance(p, Derived): - if p.loader is None: - p.loader = self.__getattribute__("_load_%s" % k) - elif isinstance(p.loader, str): - p.loader = self.__getattribute__(p.loader) - if not callable(p.loader): - raise ValueError("Callabe loader not defined for Derived object", type(self), k, p.loader) if missing: raise ValueError("One or more required properties are missing ", type(self), missing.keys()) if kwcopy: @@ -169,21 +177,3 @@ def todict(self): """ Return self cast as an '~collections.OrderedDict' object """ return odict([(key, val.todict(self)) for key, val in self._properties.items()]) - - - def _cache(self, name=None): - """ - Method called in to cache any computationally - intensive properties after updating the parameters. - - Parameters - ---------- - name : string - The parameter name. - - Returns - ------- - None - """ - #pylint: disable=unused-argument, no-self-use - return diff --git a/python/cfgmdl/derived.py b/python/cfgmdl/derived.py index 2a01ad2..35a7023 100644 --- a/python/cfgmdl/derived.py +++ b/python/cfgmdl/derived.py @@ -15,7 +15,9 @@ class Derived(Property): that is used to compute the value of the property. """ defaults = deepcopy(Property.defaults) - defaults['loader'] = (None, 'Function to load datum') + defaults['loadername'] = (None, 'Name of function to load datum') + defaults['loader'] = (None, 'Function to load datatum') + defaults['uses'] = ([], 'Properties used by this one') @classmethod def dummy(cls): #pragma: no cover @@ -24,9 +26,27 @@ def dummy(cls): #pragma: no cover @defaults_decorator(defaults) def __init__(self, **kwargs): - self.loader = self.dummy + self.loader = None + self.loadername = None + self.uses = [] super(Derived, self).__init__(**kwargs) + def __set_name__(self, owner, name): + """Set the name of the privately managed value""" + super(Derived, self).__set_name__(owner, name) + if self.loadername is None: + if self.loader is not None: + self.loadername = "_load_%s" % self.loader.__name__ + else: + self.loadername = "_load_%s" % name + if self.loader is None: + loader = getattr(owner, self.loadername, None) + self.loader = loader + if not callable(self.loader): + raise ValueError("Callable loader not defined for Derived object", owner, name, self.loadername, self.loader) + return + setattr(owner, self.loadername, self.loader) + def __get__(self, obj, objtype=None): """Get the value from the client object @@ -50,15 +70,33 @@ def __get__(self, obj, objtype=None): loader = self.loader - times = [ getattr(obj, "_%s_timestamp" % vn, 0.) for vn in loader.__func__.__code__.co_names ] + if loader is None: + raise ValueError("%s.%s no loader" % (obj, self.public_name)) + + ts_check = self.check_timestamp(obj) + if ts_check and val is not None: + return val + val = loader(obj) + self.__set__(obj, val) + return getattr(obj, self.private_name) + + def check_timestamp(self, obj): + """Check if object is up to date w.r.t. to the object it uses""" + times = [vn.timestamp(obj) for vn in self.uses] if times: ts_max = np.max(times) else: ts_max = 0. - my_ts = getattr(obj, self.time_name, -1.) + my_ts = self.timestamp(obj) + return my_ts >= ts_max - if my_ts > ts_max and val is not None: - return val - val = loader() - self.__set__(obj, val) - return getattr(obj, self.private_name) + + +def cached(**kwargs): + """Decorator attach a function to a class as a Derived property + """ + def decorator(func): + """Function that appends default kwargs to a function. + """ + return Derived(loader=func, **kwargs) + return decorator diff --git a/python/cfgmdl/function.py b/python/cfgmdl/function.py index 9d7f1aa..d906a95 100644 --- a/python/cfgmdl/function.py +++ b/python/cfgmdl/function.py @@ -27,6 +27,8 @@ def check_inputs(x, inputs): ret.append(inp(x).astype(np.float)) elif inp is None: ret.append(np.full(x.shape, np.nan)) + elif isinstance(inp, np.ndarray): + ret.append(inp) elif isinstance(inp, (Iterable, int, float)): ret.append(np.array(inp).astype(np.float)) else: diff --git a/python/cfgmdl/param_holder.py b/python/cfgmdl/param_holder.py index 86e0e37..2651e6e 100644 --- a/python/cfgmdl/param_holder.py +++ b/python/cfgmdl/param_holder.py @@ -6,6 +6,7 @@ from .array import Array from .configurable import Configurable +from .utils import is_none class ParamHolder(Configurable): """Wrapper around a data value @@ -24,9 +25,11 @@ def __init__(self, *args, **kwargs): """Constructor""" kwcopy = kwargs.copy() if args: #pragma: no cover - if 'value' in kwcopy: - raise ValueError("value keyword provided in addition to arguments") + #if 'value' in kwcopy: + # raise ValueError("value keyword provided in addition to arguments") kwcopy['value'] = args + if is_none(kwargs.get('value', None)): + kwargs.pop('value', None) super(ParamHolder, self).__init__(**kwargs) self.check_bounds() diff --git a/python/cfgmdl/parameter.py b/python/cfgmdl/parameter.py index 14b4dfc..7477083 100644 --- a/python/cfgmdl/parameter.py +++ b/python/cfgmdl/parameter.py @@ -98,7 +98,7 @@ def tostr(self, obj): """Extract values as a string""" return str(getattr(obj, self.private_name)) - def _cast_type(self, value): + def _cast_type(self, value, obj=None): """Hook took override type casting""" return value diff --git a/python/cfgmdl/property.py b/python/cfgmdl/property.py index 9d6d922..b6719ba 100644 --- a/python/cfgmdl/property.py +++ b/python/cfgmdl/property.py @@ -7,7 +7,7 @@ from collections import OrderedDict as odict -from .utils import cast_type, Meta, Defs, defaults_decorator, defaults_docstring +from .utils import is_none, cast_type, Meta, Defs, defaults_decorator, defaults_docstring try: basestring @@ -63,6 +63,7 @@ def __set_name__(self, owner, name): self.private_name = '_' + name self.time_name = '_' + name + '_timestamp' + def __set__(self, obj, value): """Set the value in the client object @@ -71,7 +72,7 @@ def __set__(self, obj, value): obj : ... The client object value : ... - The value being seti + The value being set This will use the `cast_type(self.dtype, value)` method to cast the requested value to the correct type. Rasies @@ -81,14 +82,16 @@ def __set__(self, obj, value): ValueError : The input value failes validation for a Property sub-class (e.g., not a valid choice, or outside bounds) """ try: - cast_value = self._cast_type(value) + cast_value = self._cast_type(value, obj) self.validate_value(obj, cast_value) except (TypeError, ValueError) as msg: setattr(obj, self.private_name, None) setattr(obj, self.time_name, time.time()) - raise TypeError("Failed to set %s %s" % (self.private_name, msg)) from msg + raise TypeError("Failed to set %s %s %s" % (repr(obj), self.private_name, msg)) from msg setattr(obj, self.private_name, cast_value) setattr(obj, self.time_name, time.time()) + if hasattr(obj, '_timestamp'): + setattr(obj, '_timestamp', time.time()) def __get__(self, obj, objtype=None): """Get the value from the client object @@ -104,7 +107,7 @@ def __get__(self, obj, objtype=None): The requested value """ attr = getattr(obj, self.private_name) - if self.unit is None: + if self.unit is None or is_none(attr): return attr return self.unit(attr) #pylint: disable=not-callable @@ -116,6 +119,8 @@ def __delete__(self, obj): """ setattr(obj, self.private_name, self.default) setattr(obj, self.time_name, time.time()) + if hasattr(obj, '_timestamp'): + setattr(obj, '_timestamp', time.time()) def _load(self, **kwargs): """Load kwargs key,value pairs into __dict__ @@ -136,10 +141,14 @@ def _load(self, **kwargs): _ = self._cast_type(self.default) - def _cast_type(self, value): + def _cast_type(self, value, obj=None): #pylint: disable=unused-argument """Hook took override type casting""" return cast_type(self.dtype, value) + def timestamp(self, obj): + """Get the timestamp when this object was updated""" + return max(getattr(obj, self.time_name), getattr(getattr(obj, self.private_name), '_timestamp', 0)) + @classmethod def defaults_docstring(cls, header=None, indent=None, footer=None): """Add the default values to the class docstring""" diff --git a/python/cfgmdl/ref.py b/python/cfgmdl/ref.py new file mode 100644 index 0000000..9cf6da6 --- /dev/null +++ b/python/cfgmdl/ref.py @@ -0,0 +1,78 @@ +""" Property sub-class refering to properties of other objects +""" +import time + +from copy import deepcopy + +from .property import Property, defaults_decorator + +class Ref(Property): + """Property sub-class refering to properties of other objects + """ + defaults = deepcopy(Property.defaults) + defaults.pop('unit') + defaults.pop('dtype') + defaults['owner'] = (None, 'Object that owns datum') + defaults['attr'] = (None, 'Name of attribute in object that owns datum') + + @defaults_decorator(defaults) + def __init__(self, **kwargs): + self.owner = None + self.attr = None + self.owner_name = None + self.attr = None + super(Ref, self).__init__(**kwargs) + + def __set_name__(self, owner, name): + """Set the name of the privately managed value""" + super(Ref, self).__set_name__(owner, name) + self.owner_name = '_' + name + '_owner' + if self.attr is None: + self.attr = self.public_name + + def __get__(self, obj, objtype=None): + """Get the value from the client object + + Parameter + --------- + obj : ... + The client object + + Return + ------ + val : ... + The requested value + + """ + owner = getattr(obj, self.owner_name) + if owner is None: + return None + val = getattr(owner, self.attr) + setattr(obj, self.private_name, val) + setattr(obj, self.time_name, time.time()) + return val + + def __set__(self, obj, value): + """Set the pointer to the referenced object + + Parameter + --------- + obj : ... + The client object + value : ... + The referenced object + """ + setattr(obj, self.owner_name, value) + _ = self.__get__(obj, self.public_name) + setattr(obj, self.time_name, time.time()) + + def __delete__(self, obj): + """Set the value to the default value + + This can be useful for sub-classes that use None + to indicate an un-initialized value. + """ + owner = getattr(obj, self.owner_name) + if owner is not None: + delattr(owner, self.attr) + super(Ref, self).__delete__(obj) diff --git a/python/cfgmdl/tools.py b/python/cfgmdl/tools.py new file mode 100644 index 0000000..a33420c --- /dev/null +++ b/python/cfgmdl/tools.py @@ -0,0 +1,73 @@ +"""Tools to build classes""" + +from collections import OrderedDict as odict + +from .utils import expand_dict +from .property import Property + +def build_sub_configurables(config_dict, type_dict): + """ Build a set of of Property objects to manage a set of Configurables, + + Parameters + ---------- + config_dict : `dict` + Dictionary used to build the sub-objects + + type_dict : `dict` + Dictionary mapping names to Configurable Class names + + Returns + ------- + o_dict : `dict` + odict(name:Property), i.e., a dictionary mapping names to the Property object to manage the owned objects + + Notes + ----- + This function is used by build_class to extend classes algorithmically by attaching Property objects that will be used to + access the data members of the class. + """ + + return odict([(key, Property(dtype=type_dict[val.pop('obj_type', None)])) for key, val in config_dict.items()]) + + +def build_class(name, base_classes, config_dicts, type_dicts, **kwargs): + """ Build a new class algorithmically. + + Parameters + ---------- + name : `str` + The name of the new class + + base_classes : `Iterable` + The base classes + + config_dicts : `Iterable` + Dictionaries used to build the sub-objects + + type_dicts : `Iterable` + Dictionaries mapping names to Configurable Class names for each of the config_dicts + + + Returns + ------- + object : `Configurable` + Newly-made object of the newly made type. + The kwargs are based directly to the constructor of the new class, and the resulting object is returned. + + Notes + ----- + This function is used to extend classes `Configurable` sub-classes algorithmically. + + If a `Configurable` object has some optional data members, this class will build them based + on the config_dict. + """ + kwcopy = kwargs.copy() + + props = odict() + for config_dict_, type_dict_ in zip(config_dicts, type_dicts): + use_dict_ = expand_dict(config_dict_) + kwcopy.update(use_dict_) + props.update(build_sub_configurables(use_dict_, type_dict_)) + + new_class = type(name, base_classes, props) + return new_class(**kwcopy) diff --git a/python/cfgmdl/unit.py b/python/cfgmdl/unit.py index 536c9d5..71b5f46 100644 --- a/python/cfgmdl/unit.py +++ b/python/cfgmdl/unit.py @@ -32,10 +32,14 @@ def name(self): def __call__(self, val): """Convert value to SI unit """ + if val is None: + return None return np.array(val) * self._SI def inverse(self, val): """Convert value from SI unit """ + if val is None: + return None return np.array(val) / self._SI @classmethod diff --git a/python/cfgmdl/utils.py b/python/cfgmdl/utils.py index 49def34..f5d6526 100644 --- a/python/cfgmdl/utils.py +++ b/python/cfgmdl/utils.py @@ -3,6 +3,7 @@ A few simple utilities to help parse configurations """ from collections import OrderedDict as odict +from collections.abc import Mapping, Iterable import yaml @@ -12,6 +13,33 @@ basestring = str +def expand_dict(in_dict): + """Expand a dictionary by copying defaults to a set of elements + + Parameters + ---------- + in_dict : `dict` + The input dictionary + + elem_dict : `dict` + The elements + + Returns + ------- + o_dict : `dict` + The output dict + """ + if 'default' not in in_dict: + return in_dict + default_dict = in_dict.get('default') + elem_dict = in_dict.get('elements') + o_dict = odict() + for key, elem in elem_dict.items(): + o_dict[key] = default_dict.copy() + if elem is None: + continue + o_dict[key].update(elem) + return o_dict @@ -63,7 +91,7 @@ def model_docstring(cls, header='', indent='', footer=''): #pragma: no cover #hbar = indent + width * '=' + '\n' # horizontal bar hbar = '\n' - props, _ = cls.find_properties() + props = cls.find_properties() s = hbar + (header) + hbar for key, val in props.items(): @@ -170,26 +198,35 @@ def cast_type(dtype, value): #pylint: disable=too-many-return-statements # if value is an instance of self.dtype, then return it if isinstance(value, dtype): return value - # try and cast value itself to dtype constructor + if isinstance(value, Mapping): + return dtype(**value) + if isinstance(value, Iterable): + return dtype(*value) try: return dtype(value) except (TypeError, ValueError): pass + + # try and cast value itself to dtype constructor + #try: + # return dtype(value) + #except (TypeError, ValueError): + # pass # try and cast the value as a list to dtype constructor - try: - return dtype(*value) - except (TypeError, ValueError): - pass + #try: + # return dtype(*value) + #except (TypeError, ValueError): + # pass # try and cast the value as a dict to dtype constructor - try: - return dtype(**value) - except (TypeError, ValueError): - pass + #try: + # return dtype(**value) + #except (TypeError, ValueError): + # pass # try and cast extract the 'value' item from a dict - try: - return dtype(value['value']) - except (TypeError, ValueError, KeyError): - pass + #try: + # return dtype(value['value']) + #except (TypeError, ValueError, KeyError): + # pass msg = "Value of type %s, when %s was expected." % (type(value), dtype) raise TypeError(msg) diff --git a/tests/test_derived.py b/tests/test_derived.py index f96e668..aa230b8 100644 --- a/tests/test_derived.py +++ b/tests/test_derived.py @@ -14,9 +14,9 @@ class TestClass(Model): x = Property(dtype=float, default=1., help='variable x') y = Property(dtype=float, default=2., help='variable y') z = Property(dtype=float, required=True, help='variable y') - der = Derived(dtype=float, format='%.1f', help="A derived parameter") - der2 = Derived(dtype=float, format='%.1f', loader="_loader2", help="A derived parameter") - der3 = Derived(dtype=float, format='%.1f', loader="_loader3", help="A derived parameter") + der = Derived(dtype=float, format='%.1f', uses=[x,y,z], help="A derived parameter") + der2 = Derived(dtype=float, format='%.1f', loadername="_loader2", uses=[x,y,z], help="A derived parameter") + der3 = Derived(dtype=float, format='%.1f', loadername="_loader3", uses=[], help="A derived parameter") def _load_der(self): dummy = 1. @@ -56,11 +56,11 @@ def _loader3(self): del test_obj.der assert test_obj._der is None - - class TestClass(Model): - g = 7 - der = Derived(dtype=float, format='%.1f', loader=g, help="A derived parameter") + try: + class TestClass(Model): + g = 7 + der = Derived(dtype=float, format='%.1f', loader=g, help="A derived parameter") - try: test_obj = TestClass() - except ValueError: pass - else: raise ValueError("Failed to catch ValueError in Derived.loader") + test_obj = TestClass() + except RuntimeError: pass + else: raise RuntimeError("Failed to catch ValueError in Derived.loader") diff --git a/tests/test_model.py b/tests/test_model.py index 0356156..54efc8d 100755 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -28,7 +28,7 @@ class test_class(Model): opt = Property(dtype=float, format='%.1f', default=1.0, help="An optional parameter") var = Parameter(default=1.0, bounds=[0., 3.], errors=[0.1, 0.3], free=True, help="A variable parameter") var2 = Parameter(default=1.0, free=False, help="A fixed parameter") - der = Derived(dtype=float, format='%.1f', help="A derived parameter") + der = Derived(dtype=float, uses=[req, opt, var], help="A derived parameter") def _load_der(self): return self.req * self.opt * self.var @@ -173,7 +173,7 @@ class Inner(Model): v = Property(dtype=float, default=1., help="A Property") v2 = Property(dtype=float, default=2., help="A Property") v3 = Property(dtype=float, default=3., help="A Property") - der = Derived(dtype=float, format='%.1f', help="A derived parameter") + der = Derived(dtype=float, uses=[v, v2, v3], help="A derived parameter") def _load_der(self): return self.v + self.v2 + self.v3 @@ -182,7 +182,7 @@ class TestClass(Model): p1 = Property(dtype=Inner, default=Inner()) p2 = Property(dtype=Inner, default=Inner()) px = Parameter() - der = Derived(dtype=float, format='%.1f', help="A derived parameter") + der = Derived(dtype=float, uses=[p1, p2], help="A derived parameter") def _load_der(self): return self.p1.der + self.p2.der diff --git a/tests/test_property.py b/tests/test_property.py index c6d2556..69b8606 100644 --- a/tests/test_property.py +++ b/tests/test_property.py @@ -127,7 +127,7 @@ def test_property_float(): check_property(float, 1., 2., "aa", 1, test_unit=Unit('mm')) def test_property_list(): - check_property(list, [], [3, 4], None, (3, 4), test_unit=Unit('mm')) + check_property(list, [], [3, 4], None, test_unit=Unit('mm')) def test_property_dict(): check_property(dict, {}, {3:4}) diff --git a/tests/test_units.py b/tests/test_units.py index 4372e20..ee9c33b 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -14,7 +14,9 @@ def test_units(): one = Unit() assert one.name == '' assert one(1.) == 1. - + assert one(None) is None + assert one.inverse(None) is None + mm = Unit('mm') assert mm.name == 'mm' assert mm(1.) == 1e-3