[ENG-5479] Feature/network for imps (#29)

addon_imps/__init__.py

@@ -1 +1,4 @@
-__all__ = ()
+from . import storage
+
+
+__all__ = ("storage",)

addon_imps/storage/__init__.py

@@ -0,0 +1 @@
+__all__ = ()

addon_imps/storage/my_blarg.py

@@ -1,21 +1,20 @@
 from addon_toolkit import RedirectResult
 from addon_toolkit.storage import (
-    ItemArg,
-    PageArg,
-    PagedResult,
-    StorageAddonProtocol,
+    ItemResult,
+    ItemSampleResult,
+    StorageAddonImp,
 )
 
 
-class MyBlargStorage(StorageAddonProtocol):
+class MyBlargStorage(StorageAddonImp):
     """blarg?"""
 
-    def download(self, item: ItemArg) -> RedirectResult:
+    def download(self, item_id: str) -> RedirectResult:
         """blarg blarg blarg"""
-        return RedirectResult(f"http://blarg.example/{item.item_id}")
+        return RedirectResult(f"/{item_id}")
 
-    def blargblarg(self, item: ItemArg) -> PagedResult:
-        return PagedResult(["hello"])
-
-    def opop(self, item: ItemArg, page: PageArg) -> PagedResult:
-        return PagedResult(["hello"])
+    async def get_root_items(self, page_cursor: str = "") -> ItemSampleResult:
+        return ItemSampleResult(
+            items=[ItemResult(item_id="hello", item_name="Hello!?")],
+            total_count=1,
+        )

addon_service/addon_imp/instantiation.py

@@ -0,0 +1,28 @@
+from asgiref.sync import async_to_sync
+
+from addon_service.common.aiohttp_session import get_aiohttp_client_session
+from addon_service.models import ConfiguredStorageAddon
+from addon_toolkit.constrained_aiohttp import AiohttpRequestor
+from addon_toolkit.storage import (
+    StorageAddonProtocol,
+    StorageConfig,
+)
+
+
+def get_storage_addon_instance(
+    configured_storage_addon: ConfiguredStorageAddon,
+) -> StorageAddonProtocol:
+    _external_storage_service = (
+        configured_storage_addon.base_account.external_storage_service
+    )
+    _imp_cls = _external_storage_service.addon_imp.imp_cls
+    return _imp_cls(
+        config=StorageConfig(
+            max_upload_mb=_external_storage_service.max_upload_mb,
+        ),
+        network=AiohttpRequestor(
+            client_session=async_to_sync(get_aiohttp_client_session)(),
+            prefix_url=_external_storage_service.api_base_url,
+            credentials=...,
+        ),
+    )

addon_service/addon_imp/known.py

@@ -4,11 +4,14 @@
 """
 import enum
 
-from addon_imps.storage.my_blarg import MyBlargStorage
 from addon_toolkit import AddonImp
 from addon_toolkit.storage import StorageAddonProtocol
 
 
+if __debug__:
+    from addon_imps.storage import my_blarg
+
+
 __all__ = (
     "get_imp_by_name",
     "get_imp_name",
@@ -20,15 +23,16 @@
 class KnownAddonImp(enum.Enum):
     """enum with a name for each addon implementation class that should be known to the api"""
 
-    BLARG = AddonImp(  # BLARG is fake, should be displaced by real imps soon
-        StorageAddonProtocol,
-        imp_cls=MyBlargStorage,
-        imp_number=17,
-    )
+    if __debug__:
+        BLARG = AddonImp(
+            addon_protocol_cls=StorageAddonProtocol,
+            imp_cls=my_blarg.MyBlargStorage,
+            imp_number=-7,
+        )
 
 
 ###
-# helpers using KnownAddonImp
+# helpers for accessing KnownAddonImp
 
 
 def get_imp_by_name(imp_name: str) -> AddonImp:

addon_service/addon_imp/models.py

@@ -1,7 +1,9 @@
 import dataclasses
 
+from django.utils.functional import cached_property
+
 from addon_service.addon_operation.models import AddonOperationModel
-from addon_service.common.dataclass_model import BaseDataclassModel
+from addon_service.common.static_dataclass_model import StaticDataclassModel
 from addon_toolkit import AddonImp
 
 from .known import (
@@ -13,34 +15,55 @@
 # dataclass wrapper for addon_toolkit.AddonImp that sufficiently
 # meets rest_framework_json_api expectations on a model class
 @dataclasses.dataclass(frozen=True)
-class AddonImpModel(BaseDataclassModel):
+class AddonImpModel(StaticDataclassModel):
     imp: AddonImp
 
-    @classmethod
-    def get_by_natural_key(cls, imp_name: str) -> "AddonImpModel":
-        return cls(imp=get_imp_by_name(imp_name))
+    ###
+    # class methods
 
-    @property
-    def name(self) -> str:
-        return get_imp_name(self.imp)
+    @classmethod
+    def do_get_by_natural_key(cls, *key_parts) -> "AddonImpModel":
+        (_imp_name,) = key_parts
+        return cls(get_imp_by_name(_imp_name))
 
-    @property
-    def natural_key(self) -> list:
-        return [self.name]
+    @classmethod
+    def get_model_for_imp(cls, imp: AddonImp):
+        return cls.get_by_natural_key(get_imp_name(imp))
 
-    @property
+    @cached_property
     def protocol_docstring(self) -> str:
         return self.imp.addon_protocol.protocol_cls.__doc__ or ""
 
-    @property
+    ###
+    # instance methods
+
+    @cached_property
+    def name(self) -> str:
+        return get_imp_name(self.imp)
+
+    @cached_property
+    def imp_cls(self) -> type:
+        return self.imp.imp_cls
+
+    @cached_property
     def imp_docstring(self) -> str:
         return self.imp.imp_cls.__doc__ or ""
 
+    @cached_property
+    def implemented_operations(self) -> frozenset[AddonOperationModel]:
+        return frozenset(
+            AddonOperationModel.get_model_for_operation_imp(_op_imp)
+            for _op_imp in self.imp.get_operation_imps()
+        )
+
     @property
-    def implemented_operations(self) -> list[AddonOperationModel]:
-        return [
-            AddonOperationModel(_op_imp) for _op_imp in self.imp.get_operation_imps()
-        ]
+    def natural_key(self) -> tuple[str, ...]:
+        return (self.name,)
+
+    def get_operation_imp(self, operation_name: str):
+        return AddonOperationModel.get_model_for_operation_imp(
+            self.imp.get_operation_imp_by_name(operation_name)
+        )
 
     class JSONAPIMeta:
         resource_name = "addon-imps"

addon_service/addon_imp/views.py

@@ -7,4 +7,4 @@
 
 class AddonImpViewSet(DataclassViewset):
     serializer_class = AddonImpSerializer
-    permission_classes = [AllowAny()]
+    permission_classes = [AllowAny]

addon_service/addon_operation/models.py

@@ -1,67 +1,75 @@
 import dataclasses
-import enum
+
+from django.utils.functional import cached_property
 
 from addon_service.addon_imp.known import (
     get_imp_by_name,
     get_imp_name,
 )
-from addon_service.common.dataclass_model import BaseDataclassModel
+from addon_service.common.static_dataclass_model import StaticDataclassModel
 from addon_toolkit import AddonOperationImp
 from addon_toolkit.json_arguments import jsonschema_for_signature_params
 from addon_toolkit.operation import AddonOperationType
 
 
 # dataclass wrapper for addon_toolkit.AddonOperationImp that sufficiently
 # meets rest_framework_json_api expectations on a model class
-@dataclasses.dataclass
-class AddonOperationModel(BaseDataclassModel):
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class AddonOperationModel(StaticDataclassModel):
     operation_imp: AddonOperationImp
 
     @classmethod
-    def get_by_natural_key(cls, imp_name, operation_name) -> "AddonOperationModel":
-        _addon_imp = get_imp_by_name(imp_name)
-        return cls(_addon_imp.get_operation_imp_by_name(operation_name))
-
-    @property
-    def natural_key(self) -> list:
-        return [get_imp_name(self.operation_imp.addon_imp), self.name]
+    def get_model_for_operation_imp(cls, operation_imp: AddonOperationImp):
+        return cls.get_by_natural_key(
+            get_imp_name(operation_imp.addon_imp),
+            operation_imp.declaration.name,
+        )
 
-    @property
+    @cached_property
     def name(self) -> str:
-        return self.operation_imp.operation.name
+        return self.operation_imp.declaration.name
 
-    @property
+    @cached_property
     def operation_type(self) -> AddonOperationType:
-        return self.operation_imp.operation.operation_type
+        return self.operation_imp.declaration.operation_type
 
-    @property
+    @cached_property
     def docstring(self) -> str:
-        return self.operation_imp.operation.docstring
+        return self.operation_imp.declaration.docstring
 
-    @property
+    @cached_property
     def implementation_docstring(self) -> str:
         return self.operation_imp.imp_function.__doc__ or ""
 
-    @property
-    def capability(self) -> enum.Enum:
-        return self.operation_imp.operation.capability
+    @cached_property
+    def capability(self) -> str:
+        return self.operation_imp.declaration.capability
 
-    @property
-    def imp_cls(self) -> type:
-        return self.operation_imp.addon_imp.imp_cls
+    @cached_property
+    def params_jsonschema(self) -> dict:
+        return jsonschema_for_signature_params(
+            self.operation_imp.declaration.call_signature
+        )
 
-    @property
+    @cached_property
     def implemented_by(self):
-        # avoid circular import
+        # local import to avoid circular import
         # (AddonOperationModel and AddonImpModel need to be mutually aware of each other in order to populate their respective relationship fields)
         from addon_service.addon_imp.models import AddonImpModel
 
-        return AddonImpModel(self.operation_imp.addon_imp)
+        return AddonImpModel.get_model_for_imp(self.operation_imp.addon_imp)
+
+    @classmethod
+    def do_get_by_natural_key(cls, *key_parts) -> "AddonOperationModel":
+        (_imp_name, _operation_name) = key_parts
+        _addon_imp = get_imp_by_name(_imp_name)
+        return cls(operation_imp=_addon_imp.get_operation_imp_by_name(_operation_name))
 
     @property
-    def params_jsonschema(self) -> dict:
-        return jsonschema_for_signature_params(
-            self.operation_imp.operation.call_signature
+    def natural_key(self) -> tuple[str, ...]:
+        return (
+            get_imp_name(self.operation_imp.addon_imp),
+            self.name,
         )
 
     class JSONAPIMeta:

addon_service/addon_operation/views.py

@@ -7,4 +7,4 @@
 
 class AddonOperationViewSet(DataclassViewset):
     serializer_class = AddonOperationSerializer
-    permission_classes = [AllowAny()]
+    permission_classes = [AllowAny]

addon_service/addon_operation_invocation/models.py

@@ -1,16 +1,13 @@
+import traceback
+
 import jsonschema
 from django.core.exceptions import ValidationError
-from django.db import (
-    models,
-    transaction,
-)
+from django.db import models
 
 from addon_service.common.base_model import AddonsServiceBaseModel
-from addon_service.common.dibs import dibs
 from addon_service.common.enums.validators import validate_invocation_status
 from addon_service.common.invocation import InvocationStatus
 from addon_service.models import AddonOperationModel
-from addon_toolkit.json_arguments import json_for_dataclass
 
 
 class AddonOperationInvocation(AddonsServiceBaseModel):
@@ -25,10 +22,14 @@ class AddonOperationInvocation(AddonsServiceBaseModel):
     thru_addon = models.ForeignKey("ConfiguredStorageAddon", on_delete=models.CASCADE)
     by_user = models.ForeignKey("UserReference", on_delete=models.CASCADE)
     operation_result = models.JSONField(null=True, default=None, blank=True)
+    exception_type = models.TextField(blank=True, default="")
+    exception_message = models.TextField(blank=True, default="")
+    exception_context = models.TextField(blank=True, default="")
 
     class Meta:
         indexes = [
             models.Index(fields=["operation_identifier"]),
+            models.Index(fields=["exception_type"]),
         ]
 
     class JSONAPIMeta:
@@ -46,6 +47,10 @@ def invocation_status(self, value):
     def operation(self) -> AddonOperationModel:
         return AddonOperationModel.get_by_natural_key_str(self.operation_identifier)
 
+    @property
+    def operation_name(self) -> str:
+        return self.operation.name
+
     @property
     def owner_uri(self) -> str:
         return self.by_user.user_uri
@@ -57,27 +62,17 @@ def clean_fields(self, *args, **kwargs):
                 instance=self.operation_kwargs,
                 schema=self.operation.params_jsonschema,
             )
-        except jsonschema.exceptions.ValidationError as _error:
-            raise ValidationError(_error)
+        except jsonschema.exceptions.ValidationError as _exception:
+            raise ValidationError(_exception)
+
+    def set_exception(self, exception: BaseException) -> None:
+        self.invocation_status = InvocationStatus.EXCEPTION
+        self.exception_type = type(exception).__qualname__
+        self.exception_message = repr(exception)
+        _tb = traceback.TracebackException.from_exception(exception)
+        self.exception_context = "\n".join(_tb.format(chain=True))
 
-    def perform_invocation(self, addon_instance: object):  # TODO: async_execute?
-        with dibs(self):  # TODO: handle dibs errors
-            try:
-                # wrap in a transaction to contain database errors,
-                # so status can be saved in the outer transaction
-                with transaction.atomic():
-                    _result = self.operation.operation_imp.call_with_json_kwargs(
-                        addon_instance,
-                        self.operation_kwargs,
-                    )
-            except Exception as _e:
-                self.operation_result = None
-                self.invocation_status = InvocationStatus.PROBLEM
-                print(_e)
-                # TODO: save message/traceback
-                raise
-            else:  # no errors
-                self.operation_result = json_for_dataclass(_result)
-                self.invocation_status = InvocationStatus.SUCCESS
-            finally:
-                self.save()
+    def clear_exception(self) -> None:
+        self.exception_type = ""
+        self.exception_message = ""
+        self.exception_context = ""