Merge pull request #7 from linkml/switch-to-poetry

switch to poetry

Makefile

@@ -1,4 +1,4 @@
-RUN = pipenv run
+RUN = poetry run
 
 test:
 	$(RUN) python -m unittest discover -p 'test_*.py'

linkml_dataops/changer/changer.py

@@ -1,5 +1,6 @@
 from copy import deepcopy
 from dataclasses import dataclass, field
+from typing import List
 
 from linkml_runtime.utils.formatutils import underscore
 
@@ -35,6 +36,19 @@ def apply(self, change: Change, element: YAMLRoot = None) -> ChangeResult:
         """
         raise NotImplementedError(f'{self} must implement this method')
 
+    def apply_multiple(self, changes: List[Change], element: YAMLRoot) -> List[ChangeResult]:
+        """
+        Applies multiple changes in place
+
+        :param changes:
+        :param element:
+        :return:
+        """
+        results = []
+        for change in changes:
+            results.append(self.apply(change, element, in_place=True))
+        return results
+
     def _map_change_object(self, change: YAMLRoot) -> Change:
         """
         maps a domain change object to a generic one
@@ -146,7 +160,7 @@ def _get_path(self, change: Change, element: YAMLRoot, strict=True) -> PATH_EXPR
             elif len(paths) == 1:
                 return paths[0]
             else:
-                raise Exception(f'No matching top level slot')
+                raise ValueError(f'No matching slots in element have a range of {target_cn}')
 
     def _locate_object(self, change: Change, element: YAMLRoot) -> YAMLRoot:
         """

linkml_dataops/changer/changes.yaml

@@ -105,6 +105,8 @@ classes:
       A change object that represents the addition of an object to another object
     is_a: Change
     slot_usage:
+      path:
+        required: false
       parent:
         description: the object to which the new object is being added to
       value:

linkml_dataops/changer/jsonpatch_changer.py

@@ -10,17 +10,19 @@
 from jsonpatch import JsonPatch
 
 from linkml_runtime.dumpers import json_dumper
-from linkml_runtime.linkml_model import ClassDefinitionName
+from linkml_runtime.linkml_model import ClassDefinitionName, SchemaDefinition
 from linkml_runtime.loaders import json_loader, yaml_loader
 from linkml_runtime.utils.compile_python import compile_python
+from linkml_runtime.utils.introspection import package_schemaview
 from linkml_runtime.utils.schemaview import SchemaView
+import linkml_runtime.linkml_model as linkml_model
 
 from linkml_dataops.changer.changer import Changer, ChangeResult
 from linkml_dataops.changer.changes_model import Change, AddObject, RemoveObject, Append, Rename
 from linkml_runtime.utils.formatutils import underscore
 from linkml_runtime.utils.yamlutils import YAMLRoot
 
-from linkml_dataops.changer.obj_utils import element_to_dict
+from linkml_dataops.changer.obj_utils import element_to_dict, dicts_to_changes
 from linkml_dataops.diffs.yaml_patch import YAMLPatch
 
 OPDICT = Dict[str, Any]
@@ -65,18 +67,7 @@ def apply(self, change: Change, element: YAMLRoot = None, in_place=True) -> Chan
                     setattr(element, k, getattr(new_obj, k))
         return ChangeResult(result)
 
-    def apply_multiple(self, changes: List[Change], element: YAMLRoot) -> List[ChangeResult]:
-        """
-        Applies multiple changes in place
 
-        :param changes:
-        :param element:
-        :return:
-        """
-        results = []
-        for change in changes:
-            results.append(self.apply(change, element, in_place=True))
-        return results
 
     def _change_value_as_dict(self, change: Change) -> Dict[str, Any]:
         # TODO: move this functionality into json_dumper
@@ -94,6 +85,7 @@ def make_patch(self, change: Change, element: YAMLRoot) -> OPS:
         :return:
         """
         change = self._map_change_object(change)
+        logging.info(f'Change: {change}')
         if isinstance(change, AddObject):
             return self.make_add_object_patch(change, element)
         elif isinstance(change, RemoveObject):
@@ -269,7 +261,9 @@ def patch_file(self, input_file: Union[str, IO[str]], changes: List[Change],
             obj = yaml_loader.load(input_file, target_class=target_class)
             patches = []
             for change in changes:
-                patches += self.make_patch(change, element=obj)
+                new_patches = self.make_patch(change, element=obj)
+                logging.info(f'Patches: {new_patches}')
+                patches += new_patches
                 self.apply(change, obj, in_place=True)
             # reload with rueaml, preserving comments
             yp = YAMLPatch()
@@ -332,9 +326,10 @@ def _get_format(path: str, specified_format: str =None, default=None):
     return specified_format
 
 @click.command()
+@click.option("-v", "--verbose", count=True)
 @click.option("--format", "-f", help="Input format")
 @click.option("--schema", "-S", help="Path to schema file")
-@click.option("--change-file", "-D", help="File containing yaml of changes")
+@click.option("--change-file", "-I", help="File containing yaml of changes")
 @click.option("--add", "-A", type=(str, str),
               multiple=True,
               help="add objects. List of ClassName, InitArgs dicts")
@@ -348,14 +343,30 @@ def _get_format(path: str, specified_format: str =None, default=None):
 @click.option("--target-class", "-C",
               help="name of class in datamodel that the root node instantiates")
 @click.argument('inputfile')
-def cli(inputfile, format: str, module, schema: str, change_file: str, add: List[str], remove: List[str], target_class: str, output):
+def cli(inputfile, verbose, format: str, module, schema: str, change_file: str, add: List[str], remove: List[str], target_class: str, output):
     """
     Apply changes
 
-    linkml-apply -m kitchen_sink.py -S kitchen_sink.yaml -A Person '{id: X, name: Y}' kitchen_sink_inst_01.yaml
+        linkml-apply -m kitchen_sink.py -S kitchen_sink.yaml -A Person '{id: X, name: Y}' kitchen_sink_inst_01.yaml
     """
-    python_module = compile_python(module)
-    view = SchemaView(schema)
+    if verbose >= 2:
+        logging.basicConfig(level=logging.DEBUG)
+    elif verbose == 1:
+        logging.basicConfig(level=logging.INFO)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+    #if quiet:
+    #    logging.basicConfig(level=logging.ERROR)
+    if module == 'meta':
+        python_module = linkml_model.meta
+        if target_class is None:
+            target_class = 'SchemaDefinition'
+    else:
+        python_module = compile_python(module)
+    if schema is None:
+        view = package_schemaview(python_module.__name__)
+    else:
+        view = SchemaView(schema)
     if target_class is None:
         target_class = infer_root_class(view)
     py_target_class = python_module.__dict__[target_class]
@@ -370,7 +381,8 @@ def cli(inputfile, format: str, module, schema: str, change_file: str, add: List
     changes = []
     if change_file:
         with open(change_file) as stream:
-            changes = yaml.load(stream)
+            change_dicts = yaml.safe_load(stream)
+            changes = dicts_to_changes(change_dicts, python_module)
     for (typ, ystr) in add:
         init_dict = yaml.safe_load(ystr)
         typ_cls = python_module.__dict__[typ]

linkml_dataops/changer/obj_utils.py

@@ -1,5 +1,8 @@
 import json
-from typing import Any
+import logging
+from copy import copy
+from types import ModuleType
+from typing import Any, List, Dict
 
 from jsonasobj2 import is_list, is_dict, items
 from linkml_runtime.utils.formatutils import is_empty
@@ -8,6 +11,8 @@
 # TODO: rewrite this
 from linkml_runtime.utils.yamlutils import YAMLRoot, as_json_object
 
+from linkml_dataops.changer.changes_model import AddObject, RemoveObject, Change, Rename, Append
+
 
 def to_object(obj: Any, hide_protected_keys: bool = False, inside: bool = False) -> Any:
     """
@@ -46,8 +51,50 @@ def to_object(obj: Any, hide_protected_keys: bool = False, inside: bool = False)
 
 
 def element_to_dict(element: YAMLRoot) -> dict:
-    #jsonstr = json_dumper.dumps(element, inject_type=False)
     jsonstr = json.dumps(as_json_object(element, None, inject_type=False),
                          default=lambda o: to_object(o, hide_protected_keys=True) if isinstance(o, YAMLRoot) else json.JSONDecoder().decode(o),
                          indent='  ')
-    return json.loads(jsonstr)
+    return json.loads(jsonstr)
+
+OP_PREFIX_DICT = {
+    'Add': AddObject,
+    'Remove': RemoveObject,
+    'Rename': Rename,
+    'AddTo': Append,
+}
+
+def dicts_to_changes(objs: List[Dict], python_module: ModuleType) -> List[Change]:
+    changes = []
+    for obj in objs:
+        obj = copy(obj)
+        t = obj['type']
+        del obj['type']
+        if t.startswith('Add'):
+            cc = AddObject
+            t = t.replace('Add', '', 1)
+        elif t.startswith('Remove'):
+            cc = RemoveObject
+            t = t.replace('Remove', '', 1)
+        elif t.startswith('Rename'):
+            cc = Rename
+            t = t.replace('Rename', '', 1)
+        elif t.startswith('AppendIn'):
+            cc = Append
+            t = t.replace('AppendIn', '', 1)
+        else:
+            raise ValueError(f'Unknown type: {t}')
+        typ_cls = python_module.__dict__[t]
+        v_dict = obj['value']
+        del obj['value']
+        if cc == Rename:
+            change = Rename(value=v_dict, target_class=typ_cls.class_name, **obj)
+        else:
+            if isinstance(v_dict, dict):
+                v = typ_cls(**v_dict)
+            else:
+                v = typ_cls(v_dict)
+            change = cc(value=v, **obj)
+        logging.debug(f'Created change object: {change}')
+        changes.append(change)
+    return changes
+

linkml_dataops/changer/object_changer.py

@@ -45,6 +45,12 @@ def apply(self, change: Change, element: YAMLRoot = None, in_place=True) -> Chan
 
 
     def add_object(self, change: AddObject, element: YAMLRoot) -> ChangeResult:
+        """
+
+        :param change: a change implementing AddObject
+        :param element: element
+        :return:
+        """
         place = self._locate_object(change, element)
         if isinstance(place, dict):
             pk_slot = self._get_primary_key_slot(change)
@@ -97,6 +103,7 @@ def remove_object(self, change: RemoveObject, element: YAMLRoot) -> ChangeResult
     # NOTE: changes in place
     def append_value(self, change: Append, element: YAMLRoot) -> ChangeResult:
         place = self._locate_object(change, element)
+        #print(f'Appending: {change.value} to {type(place)} {place}')
         place.append(change.value)
         return ChangeResult(object=element)
 

pyproject.toml

@@ -1,3 +1,51 @@
+[tool.poetry]
+name = "linkml-dataops"
+version = "0.0.0"
+description = "Data Operations API for the Linked Open Data Modeling Language"
+authors = [
+    "Chris Mungall <[email protected]>",
+    "Harold Solbrig <[email protected]>",
+]
+
+readme = "README.md"
+
+homepage = "https://linkml.io/linkml/"
+repository = "https://github.com/linkml/linkml-dataops"
+documentation = "https://linkml.io/linkml/"
+license = "CC0 1.0 Universal"
+
+keywords = ["schema", "linked data", "data modeling", "rdf", "owl", "biolink"]
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Healthcare Industry",
+    "License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10"
+]
+
+[tool.poetry.scripts]    
+gen-api-datamodel = "linkml_dataops.generators.apigenerator:cli"
+gen-crud-datamodel = "linkml_dataops.generators.crudmodelcreator:cli"
+gen-python-api = "linkml_dataops.generators.pyapigenerator:cli"
+linkml-apply = "linkml_dataops.changer.jsonpatch_changer:cli"
+
+[tool.poetry.dependencies]
+python = "^3.7.6"
+linkml-runtime = ">= 1.3.0"
+jsonpath-ng = "*"
+"ruamel.yaml" = "*"
+jsonpatch = "*"
+
+[tool.poetry.dev-dependencies]
+jinja2 = "*"
+
 [build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta:__legacy__"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

tests/common_tests.py

@@ -4,19 +4,21 @@
 from copy import copy
 from dataclasses import dataclass
 
+import yaml
 from linkml_runtime.dumpers import yaml_dumper
 
 from linkml_dataops.changer.changer import Changer
-from linkml_dataops.changer.object_changer import ObjectChanger
+from linkml_dataops.changer.obj_utils import dicts_to_changes
 from linkml_dataops.changer.changes_model import AddObject, RemoveObject, Append, Rename, SetValue
 from linkml_runtime.loaders import yaml_loader
-from linkml_runtime.utils.schemaview import SchemaView
+import tests.model.kitchen_sink
 from tests.model.kitchen_sink import Person, Dataset, FamilialRelationship
 from tests.model.kitchen_sink_api import AddPerson
 from tests import MODEL_DIR, INPUT_DIR
 
 SCHEMA = os.path.join(MODEL_DIR, 'kitchen_sink.yaml')
 DATA = os.path.join(INPUT_DIR, 'kitchen_sink_inst_01.yaml')
+CHANGE_FILE = os.path.join(INPUT_DIR, 'changes_01.yaml')
 
 ADD_PERSON = """
 path: /persons
@@ -200,4 +202,19 @@ def domain_api_test(self):
             if p.id == 'P:222':
                 assert p.has_familial_relationships[0].related_to == 'P:001'
                 ok = True
-        assert ok
+        assert ok
+
+    def change_file_test(self):
+        patcher = self.patcher
+        import tests.model.kitchen_sink as ks
+        with open(CHANGE_FILE) as stream:
+            changes = dicts_to_changes(yaml.safe_load(stream), ks)
+        dataset: Dataset = yaml_loader.load(DATA, target_class=Dataset)
+        patcher.apply_multiple(changes, dataset)
+        #print(yaml_dumper.dumps(dataset))
+        assert len(dataset.companies) == 1
+        assert dataset.companies[0].name == 'New Company'
+        assert len(dataset.persons) == 3
+        person_index = {p.id: p for p in dataset.persons}
+        assert person_index['P:999'].name == 'fred bloggs'
+        assert person_index['P:NEW'].name == 'New Person'