From fb67186992b2d3969fa62f7bd13238aefa73b70c Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Tue, 9 Apr 2024 16:55:43 -0400 Subject: [PATCH] [ENG-5479] Feature/network for imps (#29) --- addon_imps/__init__.py | 5 +- addon_imps/storage/__init__.py | 1 + addon_imps/storage/my_blarg.py | 23 ++- addon_service/addon_imp/instantiation.py | 28 +++ addon_service/addon_imp/known.py | 18 +- addon_service/addon_imp/models.py | 57 ++++-- addon_service/addon_imp/views.py | 2 +- addon_service/addon_operation/models.py | 68 +++---- addon_service/addon_operation/views.py | 2 +- .../addon_operation_invocation/models.py | 53 +++--- .../addon_operation_invocation/perform.py | 48 +++++ .../addon_operation_invocation/serializers.py | 15 +- .../authorized_storage_account/models.py | 2 +- addon_service/common/aiohttp_session.py | 24 +++ addon_service/common/dataclass_model.py | 52 ------ addon_service/common/enums/validators.py | 2 +- addon_service/common/invocation.py | 4 +- .../common/static_dataclass_model.py | 109 ++++++++++++ .../configured_storage_addon/models.py | 4 +- .../external_storage_service/models.py | 1 + addon_service/migrations/0001_initial.py | 12 +- addon_service/tests/_factories.py | 1 + .../test_addon_operation_invocation.py | 80 ++------- addon_toolkit/constrained_aiohttp.py | 152 ++++++++++++++++ addon_toolkit/constrained_http.py | 167 ++++++++++++++++++ addon_toolkit/cursor.py | 97 ++++++++++ addon_toolkit/imp.py | 50 ++++-- addon_toolkit/json_arguments.py | 123 ++++++++----- addon_toolkit/operation.py | 26 +-- addon_toolkit/protocol.py | 6 +- addon_toolkit/storage.py | 118 +++++++++---- addon_toolkit/tests/test_addon_protocol.py | 70 ++++++-- addon_toolkit/tests/test_constrained_http.py | 5 + requirements/requirements.txt | 2 +- 34 files changed, 1066 insertions(+), 361 deletions(-) create mode 100644 addon_imps/storage/__init__.py create mode 100644 addon_service/addon_imp/instantiation.py create mode 100644 addon_service/addon_operation_invocation/perform.py create mode 100644 addon_service/common/aiohttp_session.py delete mode 100644 addon_service/common/dataclass_model.py create mode 100644 addon_service/common/static_dataclass_model.py create mode 100644 addon_toolkit/constrained_aiohttp.py create mode 100644 addon_toolkit/constrained_http.py create mode 100644 addon_toolkit/cursor.py create mode 100644 addon_toolkit/tests/test_constrained_http.py diff --git a/addon_imps/__init__.py b/addon_imps/__init__.py index ea9b7835..663cc8c1 100644 --- a/addon_imps/__init__.py +++ b/addon_imps/__init__.py @@ -1 +1,4 @@ -__all__ = () +from . import storage + + +__all__ = ("storage",) diff --git a/addon_imps/storage/__init__.py b/addon_imps/storage/__init__.py new file mode 100644 index 00000000..ea9b7835 --- /dev/null +++ b/addon_imps/storage/__init__.py @@ -0,0 +1 @@ +__all__ = () diff --git a/addon_imps/storage/my_blarg.py b/addon_imps/storage/my_blarg.py index f4f35872..32147ad0 100644 --- a/addon_imps/storage/my_blarg.py +++ b/addon_imps/storage/my_blarg.py @@ -1,21 +1,20 @@ from addon_toolkit import RedirectResult from addon_toolkit.storage import ( - ItemArg, - PageArg, - PagedResult, - StorageAddonProtocol, + ItemResult, + ItemSampleResult, + StorageAddonImp, ) -class MyBlargStorage(StorageAddonProtocol): +class MyBlargStorage(StorageAddonImp): """blarg?""" - def download(self, item: ItemArg) -> RedirectResult: + def download(self, item_id: str) -> RedirectResult: """blarg blarg blarg""" - return RedirectResult(f"http://blarg.example/{item.item_id}") + return RedirectResult(f"/{item_id}") - def blargblarg(self, item: ItemArg) -> PagedResult: - return PagedResult(["hello"]) - - def opop(self, item: ItemArg, page: PageArg) -> PagedResult: - return PagedResult(["hello"]) + async def get_root_items(self, page_cursor: str = "") -> ItemSampleResult: + return ItemSampleResult( + items=[ItemResult(item_id="hello", item_name="Hello!?")], + total_count=1, + ) diff --git a/addon_service/addon_imp/instantiation.py b/addon_service/addon_imp/instantiation.py new file mode 100644 index 00000000..6b4ac320 --- /dev/null +++ b/addon_service/addon_imp/instantiation.py @@ -0,0 +1,28 @@ +from asgiref.sync import async_to_sync + +from addon_service.common.aiohttp_session import get_aiohttp_client_session +from addon_service.models import ConfiguredStorageAddon +from addon_toolkit.constrained_aiohttp import AiohttpRequestor +from addon_toolkit.storage import ( + StorageAddonProtocol, + StorageConfig, +) + + +def get_storage_addon_instance( + configured_storage_addon: ConfiguredStorageAddon, +) -> StorageAddonProtocol: + _external_storage_service = ( + configured_storage_addon.base_account.external_storage_service + ) + _imp_cls = _external_storage_service.addon_imp.imp_cls + return _imp_cls( + config=StorageConfig( + max_upload_mb=_external_storage_service.max_upload_mb, + ), + network=AiohttpRequestor( + client_session=async_to_sync(get_aiohttp_client_session)(), + prefix_url=_external_storage_service.api_base_url, + credentials=..., + ), + ) diff --git a/addon_service/addon_imp/known.py b/addon_service/addon_imp/known.py index bff9e9ea..2319041d 100644 --- a/addon_service/addon_imp/known.py +++ b/addon_service/addon_imp/known.py @@ -4,11 +4,14 @@ """ import enum -from addon_imps.storage.my_blarg import MyBlargStorage from addon_toolkit import AddonImp from addon_toolkit.storage import StorageAddonProtocol +if __debug__: + from addon_imps.storage import my_blarg + + __all__ = ( "get_imp_by_name", "get_imp_name", @@ -20,15 +23,16 @@ class KnownAddonImp(enum.Enum): """enum with a name for each addon implementation class that should be known to the api""" - BLARG = AddonImp( # BLARG is fake, should be displaced by real imps soon - StorageAddonProtocol, - imp_cls=MyBlargStorage, - imp_number=17, - ) + if __debug__: + BLARG = AddonImp( + addon_protocol_cls=StorageAddonProtocol, + imp_cls=my_blarg.MyBlargStorage, + imp_number=-7, + ) ### -# helpers using KnownAddonImp +# helpers for accessing KnownAddonImp def get_imp_by_name(imp_name: str) -> AddonImp: diff --git a/addon_service/addon_imp/models.py b/addon_service/addon_imp/models.py index dab718e0..a6b66827 100644 --- a/addon_service/addon_imp/models.py +++ b/addon_service/addon_imp/models.py @@ -1,7 +1,9 @@ import dataclasses +from django.utils.functional import cached_property + from addon_service.addon_operation.models import AddonOperationModel -from addon_service.common.dataclass_model import BaseDataclassModel +from addon_service.common.static_dataclass_model import StaticDataclassModel from addon_toolkit import AddonImp from .known import ( @@ -13,34 +15,55 @@ # dataclass wrapper for addon_toolkit.AddonImp that sufficiently # meets rest_framework_json_api expectations on a model class @dataclasses.dataclass(frozen=True) -class AddonImpModel(BaseDataclassModel): +class AddonImpModel(StaticDataclassModel): imp: AddonImp - @classmethod - def get_by_natural_key(cls, imp_name: str) -> "AddonImpModel": - return cls(imp=get_imp_by_name(imp_name)) + ### + # class methods - @property - def name(self) -> str: - return get_imp_name(self.imp) + @classmethod + def do_get_by_natural_key(cls, *key_parts) -> "AddonImpModel": + (_imp_name,) = key_parts + return cls(get_imp_by_name(_imp_name)) - @property - def natural_key(self) -> list: - return [self.name] + @classmethod + def get_model_for_imp(cls, imp: AddonImp): + return cls.get_by_natural_key(get_imp_name(imp)) - @property + @cached_property def protocol_docstring(self) -> str: return self.imp.addon_protocol.protocol_cls.__doc__ or "" - @property + ### + # instance methods + + @cached_property + def name(self) -> str: + return get_imp_name(self.imp) + + @cached_property + def imp_cls(self) -> type: + return self.imp.imp_cls + + @cached_property def imp_docstring(self) -> str: return self.imp.imp_cls.__doc__ or "" + @cached_property + def implemented_operations(self) -> frozenset[AddonOperationModel]: + return frozenset( + AddonOperationModel.get_model_for_operation_imp(_op_imp) + for _op_imp in self.imp.get_operation_imps() + ) + @property - def implemented_operations(self) -> list[AddonOperationModel]: - return [ - AddonOperationModel(_op_imp) for _op_imp in self.imp.get_operation_imps() - ] + def natural_key(self) -> tuple[str, ...]: + return (self.name,) + + def get_operation_imp(self, operation_name: str): + return AddonOperationModel.get_model_for_operation_imp( + self.imp.get_operation_imp_by_name(operation_name) + ) class JSONAPIMeta: resource_name = "addon-imps" diff --git a/addon_service/addon_imp/views.py b/addon_service/addon_imp/views.py index 58799936..ed1e5aa5 100644 --- a/addon_service/addon_imp/views.py +++ b/addon_service/addon_imp/views.py @@ -7,4 +7,4 @@ class AddonImpViewSet(DataclassViewset): serializer_class = AddonImpSerializer - permission_classes = [AllowAny()] + permission_classes = [AllowAny] diff --git a/addon_service/addon_operation/models.py b/addon_service/addon_operation/models.py index 3cd9e262..7e906465 100644 --- a/addon_service/addon_operation/models.py +++ b/addon_service/addon_operation/models.py @@ -1,11 +1,12 @@ import dataclasses -import enum + +from django.utils.functional import cached_property from addon_service.addon_imp.known import ( get_imp_by_name, get_imp_name, ) -from addon_service.common.dataclass_model import BaseDataclassModel +from addon_service.common.static_dataclass_model import StaticDataclassModel from addon_toolkit import AddonOperationImp from addon_toolkit.json_arguments import jsonschema_for_signature_params from addon_toolkit.operation import AddonOperationType @@ -13,55 +14,62 @@ # dataclass wrapper for addon_toolkit.AddonOperationImp that sufficiently # meets rest_framework_json_api expectations on a model class -@dataclasses.dataclass -class AddonOperationModel(BaseDataclassModel): +@dataclasses.dataclass(frozen=True, kw_only=True) +class AddonOperationModel(StaticDataclassModel): operation_imp: AddonOperationImp @classmethod - def get_by_natural_key(cls, imp_name, operation_name) -> "AddonOperationModel": - _addon_imp = get_imp_by_name(imp_name) - return cls(_addon_imp.get_operation_imp_by_name(operation_name)) - - @property - def natural_key(self) -> list: - return [get_imp_name(self.operation_imp.addon_imp), self.name] + def get_model_for_operation_imp(cls, operation_imp: AddonOperationImp): + return cls.get_by_natural_key( + get_imp_name(operation_imp.addon_imp), + operation_imp.declaration.name, + ) - @property + @cached_property def name(self) -> str: - return self.operation_imp.operation.name + return self.operation_imp.declaration.name - @property + @cached_property def operation_type(self) -> AddonOperationType: - return self.operation_imp.operation.operation_type + return self.operation_imp.declaration.operation_type - @property + @cached_property def docstring(self) -> str: - return self.operation_imp.operation.docstring + return self.operation_imp.declaration.docstring - @property + @cached_property def implementation_docstring(self) -> str: return self.operation_imp.imp_function.__doc__ or "" - @property - def capability(self) -> enum.Enum: - return self.operation_imp.operation.capability + @cached_property + def capability(self) -> str: + return self.operation_imp.declaration.capability - @property - def imp_cls(self) -> type: - return self.operation_imp.addon_imp.imp_cls + @cached_property + def params_jsonschema(self) -> dict: + return jsonschema_for_signature_params( + self.operation_imp.declaration.call_signature + ) - @property + @cached_property def implemented_by(self): - # avoid circular import + # local import to avoid circular import # (AddonOperationModel and AddonImpModel need to be mutually aware of each other in order to populate their respective relationship fields) from addon_service.addon_imp.models import AddonImpModel - return AddonImpModel(self.operation_imp.addon_imp) + return AddonImpModel.get_model_for_imp(self.operation_imp.addon_imp) + + @classmethod + def do_get_by_natural_key(cls, *key_parts) -> "AddonOperationModel": + (_imp_name, _operation_name) = key_parts + _addon_imp = get_imp_by_name(_imp_name) + return cls(operation_imp=_addon_imp.get_operation_imp_by_name(_operation_name)) @property - def params_jsonschema(self) -> dict: - return jsonschema_for_signature_params( - self.operation_imp.operation.call_signature + def natural_key(self) -> tuple[str, ...]: + return ( + get_imp_name(self.operation_imp.addon_imp), + self.name, ) class JSONAPIMeta: diff --git a/addon_service/addon_operation/views.py b/addon_service/addon_operation/views.py index 17cdc0bc..9c63c717 100644 --- a/addon_service/addon_operation/views.py +++ b/addon_service/addon_operation/views.py @@ -7,4 +7,4 @@ class AddonOperationViewSet(DataclassViewset): serializer_class = AddonOperationSerializer - permission_classes = [AllowAny()] + permission_classes = [AllowAny] diff --git a/addon_service/addon_operation_invocation/models.py b/addon_service/addon_operation_invocation/models.py index 3aa6c96a..99db3257 100644 --- a/addon_service/addon_operation_invocation/models.py +++ b/addon_service/addon_operation_invocation/models.py @@ -1,16 +1,13 @@ +import traceback + import jsonschema from django.core.exceptions import ValidationError -from django.db import ( - models, - transaction, -) +from django.db import models from addon_service.common.base_model import AddonsServiceBaseModel -from addon_service.common.dibs import dibs from addon_service.common.enums.validators import validate_invocation_status from addon_service.common.invocation import InvocationStatus from addon_service.models import AddonOperationModel -from addon_toolkit.json_arguments import json_for_dataclass class AddonOperationInvocation(AddonsServiceBaseModel): @@ -25,10 +22,14 @@ class AddonOperationInvocation(AddonsServiceBaseModel): thru_addon = models.ForeignKey("ConfiguredStorageAddon", on_delete=models.CASCADE) by_user = models.ForeignKey("UserReference", on_delete=models.CASCADE) operation_result = models.JSONField(null=True, default=None, blank=True) + exception_type = models.TextField(blank=True, default="") + exception_message = models.TextField(blank=True, default="") + exception_context = models.TextField(blank=True, default="") class Meta: indexes = [ models.Index(fields=["operation_identifier"]), + models.Index(fields=["exception_type"]), ] class JSONAPIMeta: @@ -46,6 +47,10 @@ def invocation_status(self, value): def operation(self) -> AddonOperationModel: return AddonOperationModel.get_by_natural_key_str(self.operation_identifier) + @property + def operation_name(self) -> str: + return self.operation.name + @property def owner_uri(self) -> str: return self.by_user.user_uri @@ -57,27 +62,17 @@ def clean_fields(self, *args, **kwargs): instance=self.operation_kwargs, schema=self.operation.params_jsonschema, ) - except jsonschema.exceptions.ValidationError as _error: - raise ValidationError(_error) + except jsonschema.exceptions.ValidationError as _exception: + raise ValidationError(_exception) + + def set_exception(self, exception: BaseException) -> None: + self.invocation_status = InvocationStatus.EXCEPTION + self.exception_type = type(exception).__qualname__ + self.exception_message = repr(exception) + _tb = traceback.TracebackException.from_exception(exception) + self.exception_context = "\n".join(_tb.format(chain=True)) - def perform_invocation(self, addon_instance: object): # TODO: async_execute? - with dibs(self): # TODO: handle dibs errors - try: - # wrap in a transaction to contain database errors, - # so status can be saved in the outer transaction - with transaction.atomic(): - _result = self.operation.operation_imp.call_with_json_kwargs( - addon_instance, - self.operation_kwargs, - ) - except Exception as _e: - self.operation_result = None - self.invocation_status = InvocationStatus.PROBLEM - print(_e) - # TODO: save message/traceback - raise - else: # no errors - self.operation_result = json_for_dataclass(_result) - self.invocation_status = InvocationStatus.SUCCESS - finally: - self.save() + def clear_exception(self) -> None: + self.exception_type = "" + self.exception_message = "" + self.exception_context = "" diff --git a/addon_service/addon_operation_invocation/perform.py b/addon_service/addon_operation_invocation/perform.py new file mode 100644 index 00000000..2ab5e97f --- /dev/null +++ b/addon_service/addon_operation_invocation/perform.py @@ -0,0 +1,48 @@ +from asgiref.sync import sync_to_async +from django.db import transaction + +from addon_service.addon_imp.instantiation import get_storage_addon_instance +from addon_service.common.dibs import dibs +from addon_service.common.invocation import InvocationStatus +from addon_service.models import AddonOperationInvocation +from addon_toolkit.json_arguments import json_for_typed_value + + +__all__ = ( + "perform_invocation__async", + "perform_invocation__blocking", + # TODO: @celery.task(def perform_invocation__celery) +) + + +def perform_invocation__blocking( + invocation: AddonOperationInvocation, +) -> AddonOperationInvocation: + # non-async for django transactions + with dibs(invocation): # TODO: handle dibs errors + try: + _addon_instance = get_storage_addon_instance(invocation.thru_addon) + _operation_imp = invocation.operation.operation_imp + # inner transaction to contain database errors, + # so status can be saved in the outer transaction (from `dibs`) + with transaction.atomic(): + _result = _operation_imp.call_with_json_kwargs( + _addon_instance, + invocation.operation_kwargs, + ) + invocation.operation_result = json_for_typed_value( + _operation_imp.declaration.return_type, + _result, + ) + invocation.invocation_status = InvocationStatus.SUCCESS + except BaseException as _e: + invocation.set_exception(_e) + raise # TODO: or swallow? + finally: + invocation.save() + return invocation + + +perform_invocation__async = sync_to_async(perform_invocation__blocking) +# ^ someday, this may be reversed to `async def perform_invocation__async(...)` +# and `perform_invocation__blocking = async_to_sync(perform_invocation__async)` diff --git a/addon_service/addon_operation_invocation/serializers.py b/addon_service/addon_operation_invocation/serializers.py index c52cb1d9..33d12d68 100644 --- a/addon_service/addon_operation_invocation/serializers.py +++ b/addon_service/addon_operation_invocation/serializers.py @@ -14,6 +14,8 @@ ) from addon_toolkit.operation import AddonOperationType +from .perform import perform_invocation__blocking + RESOURCE_TYPE = get_resource_type_from_model(AddonOperationInvocation) @@ -31,6 +33,7 @@ class Meta: "thru_addon", "created", "modified", + "operation_name", ] url = serializers.HyperlinkedIdentityField( @@ -41,6 +44,7 @@ class Meta: operation_result = serializers.JSONField(read_only=True) created = serializers.DateTimeField(read_only=True) modified = serializers.DateTimeField(read_only=True) + operation_name = serializers.CharField(required=True) thru_addon = ResourceRelatedField( many=False, @@ -57,6 +61,7 @@ class Meta: operation = DataclassRelatedDataField( dataclass_model=AddonOperationModel, related_link_view_name=view_names.related_view(RESOURCE_TYPE), + read_only=True, ) included_serializers = { @@ -66,7 +71,10 @@ class Meta: } def create(self, validated_data): - _operation = validated_data["operation"] + _thru_addon = validated_data["thru_addon"] + _operation_name = validated_data["operation_name"] + _addon_imp_model = _thru_addon.base_account.external_storage_service.addon_imp + _operation = _addon_imp_model.get_operation_imp(_operation_name) _invocation = AddonOperationInvocation.objects.create( operation_identifier=_operation.natural_key_str, operation_kwargs=validated_data["operation_kwargs"], @@ -75,10 +83,7 @@ def create(self, validated_data): ) match _operation.operation_type: case AddonOperationType.REDIRECT | AddonOperationType.IMMEDIATE: - _addon_instance = ( - _operation.imp_cls() - ) # TODO: consistent imp_cls instantiation (with params, probably) - _invocation.perform_invocation(_addon_instance) # block until done + perform_invocation__blocking(_invocation) case AddonOperationType.EVENTUAL: raise NotImplementedError("TODO: enqueue task") case _: diff --git a/addon_service/authorized_storage_account/models.py b/addon_service/authorized_storage_account/models.py index 0025afbc..647db7a3 100644 --- a/addon_service/authorized_storage_account/models.py +++ b/addon_service/authorized_storage_account/models.py @@ -116,7 +116,7 @@ def authorized_operations(self) -> list[AddonOperationModel]: @property def authorized_operation_names(self): return [ - _operation_imp.operation.name + _operation_imp.declaration.name for _operation_imp in self.iter_authorized_operations() ] diff --git a/addon_service/common/aiohttp_session.py b/addon_service/common/aiohttp_session.py new file mode 100644 index 00000000..c26307fa --- /dev/null +++ b/addon_service/common/aiohttp_session.py @@ -0,0 +1,24 @@ +import aiohttp + + +__all__ = ("get_aiohttp_client_session", "close_client_session") + + +__SINGLETON_CLIENT_SESSION: aiohttp.ClientSession | None = None + + +async def get_aiohttp_client_session() -> aiohttp.ClientSession: + global __SINGLETON_CLIENT_SESSION + if __SINGLETON_CLIENT_SESSION is None: + __SINGLETON_CLIENT_SESSION = aiohttp.ClientSession( + cookie_jar=aiohttp.DummyCookieJar(), # ignore all cookies + ) + return __SINGLETON_CLIENT_SESSION + + +async def close_client_session() -> None: + # TODO: figure out if/where to call this (or decide it's unnecessary) + global __SINGLETON_CLIENT_SESSION + if __SINGLETON_CLIENT_SESSION is not None: + await __SINGLETON_CLIENT_SESSION.close() + __SINGLETON_CLIENT_SESSION = None diff --git a/addon_service/common/dataclass_model.py b/addon_service/common/dataclass_model.py deleted file mode 100644 index 7e703619..00000000 --- a/addon_service/common/dataclass_model.py +++ /dev/null @@ -1,52 +0,0 @@ -import abc -from urllib.parse import ( - quote, - unquote, -) - -from addon_service.common.opaque import ( - make_opaque, - unmake_opaque, -) - - -# abstract base class for dataclasses used as models -# (put duck-typing here for rest_framework_json_api) -class BaseDataclassModel(abc.ABC): - ### - # abstract methods - - @classmethod - @abc.abstractmethod - def get_by_natural_key(cls, *key_parts): - raise NotImplementedError - - @property - @abc.abstractmethod - def natural_key(self) -> list: - raise NotImplementedError - - ### - # class methods - - @classmethod - def get_by_pk(cls, pk: str): - _natural_key_str = unmake_opaque(pk) - return cls.get_by_natural_key_str(_natural_key_str) - - @classmethod - def get_by_natural_key_str(cls, key_str: str): - _key = tuple(unquote(_key_segment) for _key_segment in key_str.split(":")) - return cls.get_by_natural_key(*_key) - - ### - # instance methods - - @property - def natural_key_str(self) -> str: - return ":".join(quote(_key_segment) for _key_segment in self.natural_key) - - @property - def pk(self) -> str: - # duck-type django.db.Model.pk - return make_opaque(self.natural_key_str) diff --git a/addon_service/common/enums/validators.py b/addon_service/common/enums/validators.py index 4ba23ce1..1ce385da 100644 --- a/addon_service/common/enums/validators.py +++ b/addon_service/common/enums/validators.py @@ -19,7 +19,7 @@ def _validate_enum_value(enum_cls, value, excluded_members=None): ### -# validators +# validators for specific controlled vocabs def validate_addon_capability(value): diff --git a/addon_service/common/invocation.py b/addon_service/common/invocation.py index 7a9f5785..7b951574 100644 --- a/addon_service/common/invocation.py +++ b/addon_service/common/invocation.py @@ -1,8 +1,8 @@ import enum -class InvocationStatus(enum.IntEnum): +class InvocationStatus(enum.Enum): STARTING = 1 GOING = 2 SUCCESS = 3 - PROBLEM = 128 + EXCEPTION = 128 diff --git a/addon_service/common/static_dataclass_model.py b/addon_service/common/static_dataclass_model.py new file mode 100644 index 00000000..1133689b --- /dev/null +++ b/addon_service/common/static_dataclass_model.py @@ -0,0 +1,109 @@ +import abc +import dataclasses +import typing +import weakref +from urllib.parse import ( + quote, + unquote, +) + +from addon_service.common.opaque import ( + make_opaque, + unmake_opaque, +) + + +NATURAL_KEY_DELIMITER = ":" + + +class StaticDataclassModel(abc.ABC): + """a django-model-like base class for statically defined, natural-keyed data + (put duck-typing here for rest_framework_json_api) + """ + + ### + # abstract methods + + @classmethod + @abc.abstractmethod + def do_get_by_natural_key(cls, *key_parts) -> typing.Self: + raise NotImplementedError + + @property + @abc.abstractmethod + def natural_key(self) -> tuple[str, ...]: + raise NotImplementedError + + ### + # class methods + + @classmethod + def get_by_pk(cls, pk: str): + _natural_key_str = unmake_opaque(pk) + return cls.get_by_natural_key_str(_natural_key_str) + + @classmethod + def get_by_natural_key_str(cls, key_str: str): + _key_parts = tuple( + unquote(_key_segment) + for _key_segment in key_str.split(NATURAL_KEY_DELIMITER) + ) + return cls.get_by_natural_key(*_key_parts) + + @classmethod + def get_by_natural_key(cls, *key_parts) -> typing.Self: + """get the static model instance for the given key + + runs `do_get_by_natural_key` once per natural key, caches result until app restart + """ + _cache = _StaticCache.for_class(cls) + try: + _gotten = _cache.by_natural_key[key_parts] + except KeyError: + _gotten = cls.do_get_by_natural_key(*key_parts) + # should already be cached, via __post_init__ + assert _cache.by_natural_key[key_parts] == _gotten + return _gotten + + ### + # instance methods + + def __post_init__(self): + self.encache_self() + + @property + def natural_key_str(self) -> str: + return NATURAL_KEY_DELIMITER.join( + quote(_key_segment) for _key_segment in self.natural_key + ) + + @property + def pk(self) -> str: + # duck-type django.db.Model.pk + return make_opaque(self.natural_key_str) + + def encache_self(self: typing.Self) -> None: + _cache = _StaticCache.for_class(self.__class__) + _cache.by_natural_key[self.natural_key] = self + + +# private caching helper + + +@dataclasses.dataclass +class _StaticCache: + by_natural_key: dict[tuple[str, ...], typing.Any] = dataclasses.field( + default_factory=dict + ) + __caches_by_class: typing.ClassVar[ + weakref.WeakKeyDictionary[type, "_StaticCache"] + ] = weakref.WeakKeyDictionary() + + @staticmethod + def for_class(any_class: type) -> "_StaticCache": + try: + return _StaticCache.__caches_by_class[any_class] + except KeyError: + _new = _StaticCache() + _StaticCache.__caches_by_class[any_class] = _new + return _new diff --git a/addon_service/configured_storage_addon/models.py b/addon_service/configured_storage_addon/models.py index 714c4f15..1d7b9c6e 100644 --- a/addon_service/configured_storage_addon/models.py +++ b/addon_service/configured_storage_addon/models.py @@ -89,14 +89,14 @@ def connected_operations(self) -> list[AddonOperationModel]: @property def connected_operation_names(self): return [ - _operation_imp.operation.name + _operation_imp.declaration.name for _operation_imp in self.iter_connected_operations() ] def iter_connected_operations(self) -> Iterator[AddonOperationImp]: _connected_caps = self.connected_capabilities for _operation_imp in self.base_account.iter_authorized_operations(): - if _operation_imp.operation.capability in _connected_caps: + if _operation_imp.declaration.capability in _connected_caps: yield _operation_imp def clean_fields(self, *args, **kwargs): diff --git a/addon_service/external_storage_service/models.py b/addon_service/external_storage_service/models.py index 05e3ca84..9a60e0f4 100644 --- a/addon_service/external_storage_service/models.py +++ b/addon_service/external_storage_service/models.py @@ -26,6 +26,7 @@ class ExternalStorageService(AddonsServiceBaseModel): max_upload_mb = models.IntegerField(null=False) auth_callback_url = models.URLField(null=False, default="") supported_scopes = ArrayField(models.CharField(), null=True, blank=True) + api_base_url = models.URLField(null=False) oauth2_client_config = models.ForeignKey( "addon_service.OAuth2ClientConfig", diff --git a/addon_service/migrations/0001_initial.py b/addon_service/migrations/0001_initial.py index ce25ee26..25be2b15 100644 --- a/addon_service/migrations/0001_initial.py +++ b/addon_service/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-04-09 13:46 +# Generated by Django 4.2.7 on 2024-04-09 18:13 import django.contrib.postgres.fields import django.db.models.deletion @@ -46,6 +46,9 @@ class Migration(migrations.Migration): "operation_result", models.JSONField(blank=True, default=None, null=True), ), + ("exception_type", models.TextField(blank=True, default="")), + ("exception_message", models.TextField(blank=True, default="")), + ("exception_context", models.TextField(blank=True, default="")), ], ), migrations.CreateModel( @@ -174,6 +177,7 @@ class Migration(migrations.Migration): base_field=models.CharField(), blank=True, null=True, size=None ), ), + ("api_base_url", models.URLField()), ], options={ "verbose_name": "External Storage Service", @@ -376,4 +380,10 @@ class Migration(migrations.Migration): fields=["operation_identifier"], name="addon_servi_operati_4bdf63_idx" ), ), + migrations.AddIndex( + model_name="addonoperationinvocation", + index=models.Index( + fields=["exception_type"], name="addon_servi_excepti_35dee4_idx" + ), + ), ] diff --git a/addon_service/tests/_factories.py b/addon_service/tests/_factories.py index e392c9c5..70d4e527 100644 --- a/addon_service/tests/_factories.py +++ b/addon_service/tests/_factories.py @@ -54,6 +54,7 @@ class Meta: max_concurrent_downloads = factory.Faker("pyint") max_upload_mb = factory.Faker("pyint") auth_callback_url = "https://osf.io/auth/callback" + api_base_url = factory.Sequence(lambda n: f"http://api.example/{n}") int_addon_imp = get_imp_by_name("BLARG").imp_number oauth2_client_config = factory.SubFactory(OAuth2ClientConfigFactory) supported_scopes = ["service.url/grant_all"] diff --git a/addon_service/tests/test_by_type/test_addon_operation_invocation.py b/addon_service/tests/test_by_type/test_addon_operation_invocation.py index b5415bd3..fd060f10 100644 --- a/addon_service/tests/test_by_type/test_addon_operation_invocation.py +++ b/addon_service/tests/test_by_type/test_addon_operation_invocation.py @@ -24,8 +24,8 @@ class TestAddonOperationInvocationCreate(APITestCase): def setUpTestData(cls): cls._configured_addon = _factories.ConfiguredStorageAddonFactory() cls._user = cls._configured_addon.base_account.account_owner - cls._operation = models.AddonOperationModel.get_by_natural_key_str( - "BLARG:blargblarg" + cls._operation = models.AddonOperationModel.get_by_natural_key( + "BLARG", "get_root_items" ) def setUp(self): @@ -43,12 +43,10 @@ def _payload_for_post(self): "data": { "type": "addon-operation-invocations", "attributes": { - "operation_kwargs": {"item": {"item_id": "foo"}}, + "operation_kwargs": {}, + "operation_name": self._operation.name, }, "relationships": { - "operation": { - "data": jsonapi_ref(self._operation), - }, "thru_addon": { "data": jsonapi_ref(self._configured_addon), }, @@ -66,9 +64,15 @@ def test_post(self): self.assertEqual( _resp.data["operation_result"], { - "item_ids": ["hello"], - "next_cursor": None, + "items": [ + { + "item_id": "hello", + "item_name": "Hello!?", + } + ], "total_count": 1, + "this_sample_cursor": "", + "first_sample_cursor": "", }, ) self.assertEqual( @@ -133,66 +137,6 @@ def test_can_load(self): self.assertEqual(self._configured_addon.pk, _resource_from_db.pk) -# unit-test viewset (call the view with test requests) -@unittest.skip("TODO") -class TestAddonOperationInvocationViewSet(TestCase): - @classmethod - def setUpTestData(cls): - cls._invocation = _factories.AddonOperationInvocationFactory() - cls._view = AddonOperationInvocationViewSet.as_view( - { - "post": "create", - "get": "retrieve", - "delete": "destroy", - } - ) - - def test_get(self): - _resp = self._view( - get_test_request(), - pk=self._invocation.pk, - ) - self.assertEqual(_resp.status_code, HTTPStatus.OK) - _content = json.loads(_resp.rendered_content) - self.assertEqual( - set(_content["data"]["attributes"].keys()), - { - "root_folder", - "connected_capabilities", - }, - ) - self.assertEqual( - _content["data"]["attributes"]["connected_capabilities"], - ["ACCESS"], - ) - self.assertEqual( - set(_content["data"]["relationships"].keys()), - { - "authorized_resource", - "base_account", - "connected_operations", - }, - ) - - @unittest.expectedFailure # TODO - def test_unauthorized(self): - _anon_resp = self._view(get_test_request(), pk=self._user.pk) - self.assertEqual(_anon_resp.status_code, HTTPStatus.UNAUTHORIZED) - - @unittest.expectedFailure # TODO - def test_wrong_user(self): - _another_user = _factories.UserReferenceFactory() - _resp = self._view( - get_test_request(user=_another_user), - pk=self._user.pk, - ) - self.assertEqual(_resp.status_code, HTTPStatus.FORBIDDEN) - - # def test_create(self): - # _post_req = get_test_request(user=self._user, method='post') - # self._view(_post_req, pk= - - @unittest.skip("TODO") class TestAddonOperationInvocationRelatedView(TestCase): @classmethod diff --git a/addon_toolkit/constrained_aiohttp.py b/addon_toolkit/constrained_aiohttp.py new file mode 100644 index 00000000..f4108639 --- /dev/null +++ b/addon_toolkit/constrained_aiohttp.py @@ -0,0 +1,152 @@ +import contextlib +import dataclasses +import logging +import typing +import weakref +from http import HTTPStatus +from urllib.parse import ( + urljoin, + urlsplit, +) + +import aiohttp # type: ignore[import-not-found] + +from addon_toolkit.constrained_http import ( + HttpRequestInfo, + HttpRequestor, + HttpResponseInfo, + Multidict, +) + + +__all__ = ("AiohttpRequestor",) + + +_logger = logging.getLogger(__name__) + + +class _AiohttpResponseInfo(HttpResponseInfo): + """an imp-friendly face for an aiohttp response (without exposing aiohttp to imps)""" + + def __init__(self, response: aiohttp.ClientResponse): + _PrivateResponse(response).assign(self) + + @property + def http_status(self) -> HTTPStatus: + _response = _PrivateResponse.get(self).aiohttp_response + return HTTPStatus(_response.status) + + @property + def headers(self) -> Multidict: + # TODO: allowed_headers config? + _response = _PrivateResponse.get(self).aiohttp_response + return Multidict(_response.headers.items()) + + async def json_content(self) -> typing.Any: + _response = _PrivateResponse.get(self).aiohttp_response + return await _response.json() + + +class AiohttpRequestor(HttpRequestor): + # abstract property from HttpRequestor: + response_info_cls = _AiohttpResponseInfo + + def __init__( + self, + *, + client_session: aiohttp.ClientSession, + prefix_url: str, + credentials: object, # TODO: base credentials? + ): + _PrivateNetworkInfo(client_session, prefix_url, credentials).assign(self) + + # abstract method from HttpRequestor: + @contextlib.asynccontextmanager + async def send(self, request: HttpRequestInfo): + _network = _PrivateNetworkInfo.get(self) + _url = _network.get_full_url(request.uri_path) + _logger.info(f"sending {request.http_method} to {_url}") + async with _network.client_session.request( + request.http_method, + _url, + headers=_network.get_headers(), + # TODO: content + ) as _response: + yield _AiohttpResponseInfo(_response) + + +### +# for info or interfaces that should not be entangled with imps + + +class _PrivateInfo: + """base class for conveniently assigning private info to an object + + >>> @dataclasses.dataclass + >>> class _MyInfo(_PrivateInfo): + ... foo: str + >>> _rando = object() + >>> _MyInfo('woo').assign(_rando) + >>> _MyInfo.get(_rando) + _MyInfo(foo='woo') + """ + + __private_map: typing.ClassVar[weakref.WeakKeyDictionary] + + def __init_subclass__(cls): + # each subclass gets its own private map -- this base class itself is unusable + cls.__private_map = weakref.WeakKeyDictionary() + + @classmethod + def get(cls, shared_obj: object): + return cls.__private_map.get(shared_obj) + + def assign(self, shared_obj: object) -> None: + self.__private_map[shared_obj] = self + + +@dataclasses.dataclass +class _PrivateResponse(_PrivateInfo): + """ "private" info associated with an _AiohttpResponseInfo instance""" + + # avoid exposing aiohttp directly to imps + aiohttp_response: aiohttp.ClientResponse + + +@dataclasses.dataclass +class _PrivateNetworkInfo(_PrivateInfo): + """ "private" info associated with an AiohttpRequestor instance""" + + # avoid exposing aiohttp directly to imps + client_session: aiohttp.ClientSession + + # keep network constraints away from imps + prefix_url: str + + # protect credentials with utmost respect + credentials: object # TODO: base credentials dataclass? (or something) + + def get_headers(self) -> Multidict: + # TODO: from self.credentials + return Multidict({"Authorization": "Bearer --"}) + + def get_full_url(self, relative_url: str) -> str: + """resolve a url relative to a given prefix + + like urllib.parse.urljoin, but return value guaranteed to start with the given `prefix_url` + """ + _split_relative = urlsplit(relative_url) + if _split_relative.scheme or _split_relative.netloc: + raise ValueError( + f'relative url may not include scheme or host (got "{relative_url}")' + ) + if _split_relative.path.startswith("/"): + raise ValueError( + f'relative url may not be an absolute path starting with "/" (got "{relative_url}")' + ) + _full_url = urljoin(self.prefix_url, relative_url) + if not _full_url.startswith(self.prefix_url): + raise ValueError( + f'relative url may not alter the base url (maybe with dot segments "/../"? got "{relative_url}")' + ) + return _full_url diff --git a/addon_toolkit/constrained_http.py b/addon_toolkit/constrained_http.py new file mode 100644 index 00000000..132c5b52 --- /dev/null +++ b/addon_toolkit/constrained_http.py @@ -0,0 +1,167 @@ +import contextlib +import dataclasses +import typing +from collections.abc import ( + Iterable, + Mapping, +) +from functools import partialmethod +from http import ( + HTTPMethod, + HTTPStatus, +) +from urllib.parse import quote +from wsgiref.headers import Headers + + +KeyValuePairs = Iterable[tuple[str, str]] | Mapping[str, str] + + +class Multidict(Headers): + """multidict with string keys and string values, for iri queries or request headers + + can initialize from an iterable of key-value pairs: + >>> Multidict([('a', 'a1'), ('a', 'a2')]).as_query_string() + 'a=a1&a=a2' + + ...or from a mapping (with single values only): + >>> Multidict({'a': 'aval', 'b': 'bval'}).as_query_string() + 'a=aval&b=bval' + + ...or empty: + >>> Multidict().as_query_string() + '' + + use `_.add(key, value)` to add an entry, allowing for multiple values per key + >>> _m = Multidict([('a', 'two'), ('7', '🐙')]) + >>> _m.add('a', 'three') + >>> _m.as_query_string() + 'a=two&7=%F0%9F%90%99&a=three' + + use set-item syntax (`_[key] = value`) to overwrite existing values + >>> _m['a'] = 'five' + >>> _m.as_query_string() + '7=%F0%9F%90%99&a=five' + + inherits `wsgiref.headers.Headers`, a string multidict conveniently in the standard library + https://docs.python.org/3/library/wsgiref.html#wsgiref.headers.Headers + """ + + def __init__(self, key_value_pairs: KeyValuePairs | None = None): + # allow initializing with any iterable or mapping type (`Headers` expects `list`) + match key_value_pairs: + case None: + _headerslist = [] + case list(): # already a list, is fine + _headerslist = key_value_pairs + case Mapping(): + _headerslist = list(key_value_pairs.items()) + case _: # assume iterable + _headerslist = list(key_value_pairs) + super().__init__(_headerslist) + + def add(self, key: str, value: str, **mediatype_params): + """add a key-value pair (allowing other values to exist) + + alias of `wsgiref.headers.Headers.add_header` + """ + super().add_header(key, value, **mediatype_params) + + def as_headers(self) -> bytes: + """format as http headers + + same as calling `bytes()` on a `wsgiref.headers.Headers` object -- see + https://docs.python.org/3/library/wsgiref.html#wsgiref.headers.Headers + """ + return super().__bytes__() + + def as_query_string(self) -> str: + """format as query string, url-quoting parameter names and values""" + return "&".join( + "=".join((quote(_param_name), quote(_param_value))) + for _param_name, _param_value in self.items() + ) + + +@dataclasses.dataclass +class HttpRequestInfo: + http_method: HTTPMethod + uri_path: str + query: Multidict + headers: Multidict + + # TODO: content (when needed) + + +class HttpResponseInfo(typing.Protocol): + @property + def http_status(self) -> HTTPStatus: + ... + + @property + def headers(self) -> Multidict: + ... + + async def json_content(self) -> typing.Any: + ... + + # TODO: streaming (when needed) + + +class _MethodRequestMethod(typing.Protocol): + """structural type for the convenience methods HttpRequestor has per http method + + (name is joke: "method" has different but colliding meanings in http and python) + """ + + def __call__( + self, + uri_path: str, + query: Multidict | KeyValuePairs | None = None, + headers: Multidict | KeyValuePairs | None = None, + ) -> contextlib.AbstractAsyncContextManager[HttpResponseInfo]: + ... + + +class HttpRequestor(typing.Protocol): + @property + def response_info_cls(self) -> type[HttpResponseInfo]: + ... + + # abstract method for subclasses + def send( + self, + request: HttpRequestInfo, + ) -> contextlib.AbstractAsyncContextManager[HttpResponseInfo]: + ... + + @contextlib.asynccontextmanager + async def request( + self, + http_method: HTTPMethod, + uri_path: str, + query: Multidict | KeyValuePairs | None = None, + headers: Multidict | KeyValuePairs | None = None, + ): + _request_info = HttpRequestInfo( + http_method=http_method, + uri_path=uri_path, + query=(query if isinstance(query, Multidict) else Multidict(query)), + headers=(headers if isinstance(headers, Multidict) else Multidict(headers)), + ) + async with self.send(_request_info) as _response: + yield _response + + # TODO: streaming send/receive (only if/when needed) + + ### + # convenience methods for http methods + # (same call signature as self.request, minus `http_method`) + + OPTIONS: _MethodRequestMethod = partialmethod(request, HTTPMethod.OPTIONS) + HEAD: _MethodRequestMethod = partialmethod(request, HTTPMethod.HEAD) + GET: _MethodRequestMethod = partialmethod(request, HTTPMethod.GET) + PATCH: _MethodRequestMethod = partialmethod(request, HTTPMethod.PATCH) + POST: _MethodRequestMethod = partialmethod(request, HTTPMethod.POST) + PUT: _MethodRequestMethod = partialmethod(request, HTTPMethod.PUT) + DELETE: _MethodRequestMethod = partialmethod(request, HTTPMethod.DELETE) diff --git a/addon_toolkit/cursor.py b/addon_toolkit/cursor.py new file mode 100644 index 00000000..b2d10374 --- /dev/null +++ b/addon_toolkit/cursor.py @@ -0,0 +1,97 @@ +import base64 +import dataclasses +import json +from typing import ( + ClassVar, + Protocol, +) + + +def encode_cursor_dataclass(dataclass_instance) -> str: + _as_json = json.dumps(dataclasses.astuple(dataclass_instance)) + _cursor_bytes = base64.b64encode(_as_json.encode()) + return _cursor_bytes.decode() + + +def decode_cursor_dataclass(cursor: str, dataclass_class): + _as_list = json.loads(base64.b64decode(cursor)) + return dataclass_class(*_as_list) + + +class Cursor(Protocol): + @classmethod + def from_str(cls, cursor: str): + return decode_cursor_dataclass(cursor, cls) + + @property + def this_cursor_str(self) -> str: + return encode_cursor_dataclass(self) + + @property + def next_cursor_str(self) -> str | None: + ... + + @property + def prev_cursor_str(self) -> str | None: + ... + + @property + def first_cursor_str(self) -> str: + ... + + @property + def is_first_page(self) -> bool: + ... + + @property + def is_last_page(self) -> bool: + ... + + @property + def has_many_more(self) -> bool: + ... + + +@dataclasses.dataclass +class OffsetCursor(Cursor): + offset: int + limit: int + total_count: int # use -1 to mean "many more" + + MAX_INDEX: ClassVar[int] = 9999 + + @property + def next_cursor_str(self) -> str | None: + _next = dataclasses.replace(self, offset=(self.offset + self.limit)) + return encode_cursor_dataclass(_next) if _next.is_valid_cursor() else None + + @property + def prev_cursor_str(self) -> str | None: + _prev = dataclasses.replace(self, offset=(self.offset - self.limit)) + return encode_cursor_dataclass(_prev) if _prev.is_valid_cursor() else None + + @property + def first_cursor_str(self) -> str: + return encode_cursor_dataclass(dataclasses.replace(self, offset=0)) + + @property + def is_first_page(self) -> bool: + return self.offset == 0 + + @property + def is_last_page(self) -> bool: + return (self.offset + self.limit) >= self.total_count + + @property + def has_many_more(self) -> bool: + return self.total_count == -1 + + def max_index(self) -> int: + return ( + self.MAX_INDEX + if self.has_many_more + else min(self.total_count, self.MAX_INDEX) + ) + + def is_valid_cursor(self) -> bool: + return (self.limit > 0) and (0 <= self.offset < self.max_index()) diff --git a/addon_toolkit/imp.py b/addon_toolkit/imp.py index 14f2da02..13279ce8 100644 --- a/addon_toolkit/imp.py +++ b/addon_toolkit/imp.py @@ -1,11 +1,14 @@ import dataclasses import enum +import inspect from typing import ( Iterable, Iterator, ) -from .json_arguments import bound_kwargs_from_json +from asgiref.sync import async_to_sync + +from .json_arguments import kwargs_from_json from .operation import AddonOperationDeclaration from .protocol import ( AddonProtocolDeclaration, @@ -29,7 +32,8 @@ class AddonImp: addon_protocol: AddonProtocolDeclaration = dataclasses.field(init=False) def __post_init__(self, addon_protocol_cls): - super().__setattr__( # using __setattr__ to bypass dataclass frozenness + object.__setattr__( # using __setattr__ to bypass dataclass frozenness + self, "addon_protocol", addon_protocol.get_declaration(addon_protocol_cls), ) @@ -37,9 +41,11 @@ def __post_init__(self, addon_protocol_cls): def get_operation_imps( self, *, capabilities: Iterable[enum.Enum] = () ) -> Iterator["AddonOperationImp"]: - for _operation in self.addon_protocol.get_operations(capabilities=capabilities): + for _declaration in self.addon_protocol.get_operation_declarations( + capabilities=capabilities + ): try: - yield AddonOperationImp(addon_imp=self, operation=_operation) + yield AddonOperationImp(addon_imp=self, declaration=_declaration) except NotImplementedError: # TODO: helpful exception type pass # operation not implemented @@ -47,7 +53,9 @@ def get_operation_imp_by_name(self, operation_name: str) -> "AddonOperationImp": try: return AddonOperationImp( addon_imp=self, - operation=self.addon_protocol.get_operation_by_name(operation_name), + declaration=self.addon_protocol.get_operation_declaration_by_name( + operation_name + ), ) except NotImplementedError: # TODO: helpful exception type raise ValueError(f'unknown operation name "{operation_name}"') @@ -58,31 +66,35 @@ class AddonOperationImp: """dataclass for an operation implemented as part of an addon protocol implementation""" addon_imp: AddonImp - operation: AddonOperationDeclaration + declaration: AddonOperationDeclaration def __post_init__(self): _protocol_fn = getattr( - self.addon_imp.addon_protocol.protocol_cls, self.operation.name + self.addon_imp.addon_protocol.protocol_cls, self.declaration.name ) - if self.imp_function is _protocol_fn: + try: + _imp_fn = self.imp_function + except AttributeError: + _imp_fn = _protocol_fn + if _imp_fn is _protocol_fn: raise NotImplementedError( # TODO: helpful exception type - f"operation '{self.operation}' not implemented by {self.addon_imp}" + f"operation '{self.declaration}' not implemented by {self.addon_imp}" ) @property def imp_function(self): - return getattr(self.addon_imp.imp_cls, self.operation.name) + return getattr(self.addon_imp.imp_cls, self.declaration.name) def call_with_json_kwargs(self, addon_instance: object, json_kwargs: dict): - assert isinstance(addon_instance, self.addon_imp.imp_cls) - _bound_kwargs = bound_kwargs_from_json( - self.operation.call_signature, json_kwargs - ) - _method = getattr(addon_instance, self.operation.name) - _result = _method( - *_bound_kwargs.args, **_bound_kwargs.kwargs - ) # TODO: if async, use async_to_sync - assert isinstance(_result, self.operation.return_dataclass) + _method = self._get_instance_method(addon_instance) + _kwargs = kwargs_from_json(self.declaration.call_signature, json_kwargs) + if inspect.iscoroutinefunction(_method): + _method = async_to_sync(_method) + _result = _method(**_kwargs) + assert isinstance(_result, self.declaration.return_type) return _result + def _get_instance_method(self, addon_instance: object): + return getattr(addon_instance, self.declaration.name) + # TODO: async def async_call_with_json_kwargs(self, addon_instance: object, json_kwargs: dict): diff --git a/addon_toolkit/json_arguments.py b/addon_toolkit/json_arguments.py index 62a16801..f5ce66ee 100644 --- a/addon_toolkit/json_arguments.py +++ b/addon_toolkit/json_arguments.py @@ -6,10 +6,7 @@ __all__ = ( - "bound_kwargs_from_json", - "dataclass_from_json", - "json_for_arguments", - "json_for_dataclass", + "kwargs_from_json", "json_for_typed_value", "jsonschema_for_annotation", "jsonschema_for_dataclass", @@ -26,7 +23,6 @@ def jsonschema_for_signature_params(signature: inspect.Signature) -> dict: 'properties': {'a': {'type': 'string'}, 'b': {'type': 'number'}}, 'required': ['a']} """ - # TODO: required/optional fields return { "type": "object", "properties": { @@ -43,7 +39,7 @@ def jsonschema_for_signature_params(signature: inspect.Signature) -> dict: def jsonschema_for_dataclass(dataclass: type) -> dict: - """build jsonschema corresponding to fields in a dataclass""" + """build jsonschema corresponding to the constructor signature of a dataclass""" assert dataclasses.is_dataclass(dataclass) and isinstance(dataclass, type) return jsonschema_for_signature_params(inspect.signature(dataclass)) @@ -63,17 +59,6 @@ def jsonschema_for_annotation(annotation: type) -> dict: raise NotImplementedError(f"what do with param annotation '{annotation}'?") -def json_for_arguments(bound_kwargs: inspect.BoundArguments) -> dict: - """return json-serializable representation of the dataclass instance""" - return { - _param_name: json_for_typed_value( - bound_kwargs.signature.parameters[_param_name].annotation, - _arg_value, - ) - for (_param_name, _arg_value) in bound_kwargs.arguments.items() - } - - # TODO generic type: def json_for_typed_value[_ValueType: object](type_annotation: type[_ValueType], value: _ValueType): def json_for_typed_value(type_annotation: typing.Any, value: typing.Any): """return json-serializable representation of field value @@ -85,25 +70,27 @@ def json_for_typed_value(type_annotation: typing.Any, value: typing.Any): >>> json_for_typed_value(list[int], [2,3,'7']) [2, 3, 7] """ + _is_optional, _type = _maybe_optional_type(type_annotation) if value is None: - # check type_annotation allows None - assert isinstance(None, type_annotation), f"got {value=} with {type_annotation}" + if not _is_optional: + raise ValueError(f"got `None` for non-optional type {type_annotation}") return None - if dataclasses.is_dataclass(type_annotation): - if not isinstance(value, type_annotation): - raise ValueError(f"expected instance of {type_annotation}, got {value}") + if dataclasses.is_dataclass(_type): + if not isinstance(value, _type): + raise ValueError(f"expected instance of {_type}, got {value}") return json_for_dataclass(value) - if issubclass(type_annotation, enum.Enum): - if value not in type_annotation: - raise ValueError(f"expected member of enum {type_annotation}, got {value}") + if issubclass(_type, enum.Enum): + if value not in _type: + raise ValueError(f"expected member of enum {_type}, got {value}") return value.value - if type_annotation in (str, int, float): # check str before Iterable - return type_annotation(value) - # support parameterized generics like `list[int]` - if isinstance(type_annotation, types.GenericAlias): - if issubclass(type_annotation.__origin__, typing.Iterable): + if _type in (str, int, float): # check str before Iterable + return _type(value) + if isinstance(_type, types.GenericAlias): + # parameterized generic like `list[int]` + if issubclass(_type.__origin__, typing.Iterable): + # iterables the only supported generic (...yet) try: - (_item_annotation,) = type_annotation.__args__ + (_item_annotation,) = _type.__args__ except ValueError: pass else: @@ -111,19 +98,28 @@ def json_for_typed_value(type_annotation: typing.Any, value: typing.Any): json_for_typed_value(_item_annotation, _item_value) for _item_value in value ] - raise NotImplementedError( - f"what do with argument type {type_annotation}? ({value=})" - ) - - -def bound_kwargs_from_json( - signature: inspect.Signature, args_from_json: dict -) -> inspect.BoundArguments: - _kwargs = { - _param_name: arg_value_from_json(signature.parameters[_param_name], _arg_value) - for (_param_name, _arg_value) in args_from_json.items() - } - return signature.bind_partial(**_kwargs) + raise NotImplementedError(f"what do with argument type {_type}? ({value=})") + + +def kwargs_from_json( + signature: inspect.Signature, + args_from_json: dict, +) -> dict: + try: + _kwargs = { + _param_name: arg_value_from_json( + signature.parameters[_param_name], _arg_value + ) + for (_param_name, _arg_value) in args_from_json.items() + } + # use inspect.Signature.bind() (with dummy `self` value) to validate all required kwargs present + _bound_kwargs = signature.bind(self=..., **_kwargs) + except (TypeError, KeyError): + raise ValueError( + f"invalid json args for signature\n{signature=}\nargs={args_from_json}" + ) + _bound_kwargs.arguments.pop("self") + return _bound_kwargs.arguments def arg_value_from_json( @@ -146,11 +142,14 @@ def arg_value_from_json( def json_for_dataclass(dataclass_instance) -> dict: """return json-serializable representation of the dataclass instance""" - return { - _field.name: json_for_typed_value( - _field.type, getattr(dataclass_instance, _field.name) - ) + _field_value_pairs = ( + (_field, getattr(dataclass_instance, _field.name)) for _field in dataclasses.fields(dataclass_instance) + ) + return { + _field.name: json_for_typed_value(_field.type, _value) + for _field, _value in _field_value_pairs + if (_value is not None) or (_field.default is not None) } @@ -178,3 +177,31 @@ def field_value_from_json(field: dataclasses.Field, dataclass_json: dict): assert isinstance(_json_value, field.type) return _json_value raise NotImplementedError(f"what do with {_json_value=} (value for {field})") + + +def _maybe_optional_type(type_annotation: typing.Any) -> tuple[bool, typing.Any]: + """given a type annotation, detect whether it allows `None` and extract the non-optional type + + >>> _maybe_optional_type(int) + (False, ) + >>> _maybe_optional_type(int | None) + (True, ) + >>> _maybe_optional_type(None) + (True, None) + """ + if isinstance(type_annotation, types.UnionType): + _allows_none = type(None) in type_annotation.__args__ + _nonnone_types = [ + _alt_type + for _alt_type in type_annotation.__args__ + if _alt_type is not type(None) + ] + _base_type = ( + _nonnone_types[0] + if len(_nonnone_types) == 1 + else types.UnionType(*_nonnone_types) + ) + else: + _allows_none = type_annotation is None + _base_type = type_annotation + return (_allows_none, _base_type) diff --git a/addon_toolkit/operation.py b/addon_toolkit/operation.py index a29500e5..75efe879 100644 --- a/addon_toolkit/operation.py +++ b/addon_toolkit/operation.py @@ -37,7 +37,7 @@ class AddonOperationDeclaration: operation_type: AddonOperationType capability: enum.Enum operation_fn: Callable # the decorated function - return_dataclass: type = dataclasses.field( + return_type: type = dataclasses.field( default=type(None), # if not provided, inferred by __post_init__ compare=False, ) @@ -48,32 +48,32 @@ def for_function(self, fn: Callable) -> "AddonOperationDeclaration": def __post_init__(self): _return_type = self.call_signature.return_annotation - if self.return_dataclass is type(None): - # no return_dataclass declared; infer from type annotation + if self.return_type is type(None): + # no return_type declared; infer from type annotation assert dataclasses.is_dataclass( _return_type ), f"operation methods must return a dataclass (got {_return_type} on {self.operation_fn})" - # use __setattr__ to bypass dataclass frozenness (only here in __post_init__) - super().__setattr__("return_dataclass", _return_type) + # use object.__setattr__ to bypass dataclass frozenness (only here in __post_init__) + object.__setattr__(self, "return_type", _return_type) else: - # return_dataclass declared; enforce it + # return_type declared; enforce it assert dataclasses.is_dataclass( - self.return_dataclass - ), f"return_dataclass must be a dataclass (got {self.return_dataclass})" - if not issubclass(_return_type, self.return_dataclass): + self.return_type + ), f"return_type must be a dataclass (got {self.return_type})" + if not issubclass(_return_type, self.return_type): raise ValueError( - f"expected return type {self.return_dataclass} on operation function {self.operation_fn} (got {_return_type})" + f"expected return type {self.return_type} on operation function {self.operation_fn} (got {_return_type})" ) @property def name(self): - # TODO: language tag + # TODO: language tag (kwarg for tagged string?) return self.operation_fn.__name__ @property def docstring(self) -> str: # TODO: language tag - # TODO: consider docstring param on operation decorators, allow overriding __doc__ + # TODO: docstring/description param on operation decorators, since __doc__ is removed on -O return self.operation_fn.__doc__ or "" @property @@ -105,7 +105,7 @@ class RedirectResult: # decorator for operations that may be performed by a client request (e.g. redirect to waterbutler) redirect_operation = addon_operation.with_kwargs( operation_type=AddonOperationType.REDIRECT, - return_dataclass=RedirectResult, + return_type=RedirectResult, # TODO: consider adding `save_invocation: bool = True`, set False here ) diff --git a/addon_toolkit/protocol.py b/addon_toolkit/protocol.py index 5ba0b613..f0e2c603 100644 --- a/addon_toolkit/protocol.py +++ b/addon_toolkit/protocol.py @@ -28,7 +28,7 @@ class AddonProtocolDeclaration: protocol_cls: type - def get_operations( + def get_operation_declarations( self, *, capabilities: Iterable[enum.Enum] = () ) -> Iterator[AddonOperationDeclaration]: _capability_set = set(capabilities) @@ -40,7 +40,9 @@ def get_operations( if (not _capability_set) or (_op.capability in _capability_set): yield _op - def get_operation_by_name(self, op_name: str) -> AddonOperationDeclaration: + def get_operation_declaration_by_name( + self, op_name: str + ) -> AddonOperationDeclaration: return addon_operation.get_declaration( getattr(self.protocol_cls, op_name), ) diff --git a/addon_toolkit/storage.py b/addon_toolkit/storage.py index 38079dec..85e605ac 100644 --- a/addon_toolkit/storage.py +++ b/addon_toolkit/storage.py @@ -1,4 +1,6 @@ -"""what a base StorageAddonProtocol could be like (incomplete)""" +"""a static (and still in progress) definition of what composes a storage addon""" + +import collections.abc import dataclasses import typing @@ -10,48 +12,83 @@ immediate_operation, redirect_operation, ) +from addon_toolkit.constrained_http import HttpRequestor +from addon_toolkit.cursor import Cursor + + +__all__ = ( + "ItemResult", + "ItemSampleResult", + "PathResult", + "PossibleSingleItemResult", + "StorageAddonImp", + "StorageAddonProtocol", + "StorageConfig", +) -__all__ = ("StorageAddonProtocol",) +### +# dataclasses used for operation args and return values -### -# use dataclasses for operation args and return values +@dataclasses.dataclass(frozen=True) +class StorageConfig: + max_upload_mb: int @dataclasses.dataclass -class PagedResult: - item_ids: list[str] - total_count: int = 0 - next_cursor: str | None = None +class ItemResult: + item_id: str + item_name: str + item_path: list[str] | None = None - def __post_init__(self): - if (self.total_count == 0) and (self.next_cursor is None) and self.item_ids: - self.total_count = len(self.item_ids) + +@dataclasses.dataclass +class PathResult: + ancestor_ids: collections.abc.Sequence[str] # most distant first @dataclasses.dataclass -class PageArg: - cursor: str = "" +class PossibleSingleItemResult: + possible_item: ItemResult | None @dataclasses.dataclass -class ItemArg: - item_id: str +class ItemSampleResult: + """a sample from a possibly-large population of result items""" + + items: collections.abc.Collection[ItemResult] + total_count: int | None = None + this_sample_cursor: str = "" + next_sample_cursor: str | None = None # when None, this is the last page of results + prev_sample_cursor: str | None = None + first_sample_cursor: str = "" + # optional init var: + cursor: dataclasses.InitVar[Cursor | None] = None -@addon_protocol() + def __post_init__(self, cursor: Cursor | None): + if cursor is not None: + self.this_sample_cursor = cursor.this_cursor_str + self.next_sample_cursor = cursor.next_cursor_str + self.prev_sample_cursor = cursor.prev_cursor_str + self.first_sample_cursor = cursor.first_cursor_str + + +### +# use python's typing.Protocol to define a shared interface for storage addons + + +@addon_protocol() # TODO: descriptions with language tags class StorageAddonProtocol(typing.Protocol): - @redirect_operation(capability=AddonCapabilities.ACCESS) - def download(self, item: ItemArg) -> RedirectResult: + def __init__(self, config: StorageConfig, network: HttpRequestor): ... - @immediate_operation(capability=AddonCapabilities.ACCESS) - def blargblarg(self, item: ItemArg) -> PagedResult: - ... + ### + # declared operations: - @immediate_operation(capability=AddonCapabilities.ACCESS) - def opop(self, item: ItemArg, page: PageArg) -> PagedResult: + @redirect_operation(capability=AddonCapabilities.ACCESS) + def download(self, item_id: str) -> RedirectResult: ... # @@ -72,18 +109,23 @@ def opop(self, item: ItemArg, page: PageArg) -> PagedResult: # "tree-read" operations: @immediate_operation(capability=AddonCapabilities.ACCESS) - async def get_root_item_ids(self, page: PageArg) -> PagedResult: + async def get_root_items(self, page_cursor: str = "") -> ItemSampleResult: ... - # - # @immediate_operation(capability=AddonCapabilities.ACCESS) - # async def get_parent_item_id(self, item_id: str) -> str | None: ... - # - # @immediate_operation(capability=AddonCapabilities.ACCESS) - # async def get_item_path(self, item_id: str) -> str: ... - # @immediate_operation(capability=AddonCapabilities.ACCESS) - async def get_child_item_ids(self, item: ItemArg, page: PageArg) -> PagedResult: + async def get_parent_item_id(self, item_id: str) -> PossibleSingleItemResult: + ... + + @immediate_operation(capability=AddonCapabilities.ACCESS) + async def get_item_path(self, item_id: str) -> PathResult: + ... + + @immediate_operation(capability=AddonCapabilities.ACCESS) + async def get_child_items( + self, + item_id: str, + page_cursor: str = "", + ) -> ItemSampleResult: ... @@ -111,3 +153,15 @@ async def get_child_item_ids(self, item: ItemArg, page: PageArg) -> PagedResult: # # @immediate_operation(capability=AddonCapabilities.UPDATE) # async def pls_restore_version(self, item_id: str, version_id: str): ... + + +@dataclasses.dataclass(frozen=True) +class StorageAddonImp(StorageAddonProtocol): + """a still-abstract implementation of StorageAddonProtocol + + storage-addon implementations should probably inherit this + and start implementing operation methods + """ + + config: StorageConfig + network: HttpRequestor diff --git a/addon_toolkit/tests/test_addon_protocol.py b/addon_toolkit/tests/test_addon_protocol.py index d6773120..f3cf470c 100644 --- a/addon_toolkit/tests/test_addon_protocol.py +++ b/addon_toolkit/tests/test_addon_protocol.py @@ -3,6 +3,7 @@ import typing import unittest from http import HTTPMethod +from unittest.mock import Mock from addon_toolkit import ( AddonImp, @@ -35,7 +36,7 @@ class _MyCapability(enum.Enum): _expected_put_imp: AddonOperationImp @classmethod - def setUpClass(cls): + def setUpClass(cls) -> None: ### # declare the capabilities and protocol for a category of addons @@ -49,19 +50,21 @@ class _MyProtocol(typing.Protocol): """this _MyProtocol docstring should find its way to browsable docs somewhere""" @redirect_operation(capability=cls._MyCapability.GET_IT) - def url_for_get(self, checksum_iri) -> RedirectResult: + def url_for_get(self, checksum_iri: str) -> RedirectResult: """this url_for_get docstring should find its way to docs""" ... @immediate_operation(capability=cls._MyCapability.GET_IT) async def query_relations( - self, checksum_iri, query=None + self, + checksum_iri: str, + query: str | None = None, ) -> _MyCustomOperationResult: """this query_relations docstring should find its way to docs""" ... @redirect_operation(capability=cls._MyCapability.PUT_IT) - def url_for_put(self, checksum_iri) -> RedirectResult: + def url_for_put(self, checksum_iri: str) -> RedirectResult: """this url_for_put docstring should find its way to docs""" ... @@ -69,18 +72,18 @@ def url_for_put(self, checksum_iri) -> RedirectResult: # implement (some of) the protocol's declared operations class _MyImplementation(_MyProtocol): - def url_for_get(self, checksum_iri) -> RedirectResult: + def url_for_get(self, checksum_iri: str) -> RedirectResult: """this url_for_get docstring could contain implementation-specific caveats""" return RedirectResult( - HTTPMethod.GET, f"https://myarchive.example///{checksum_iri}", + HTTPMethod.GET, ) - def url_for_put(self, checksum_iri) -> RedirectResult: + def url_for_put(self, checksum_iri: str) -> RedirectResult: """this url_for_put docstring could contain implementation-specific caveats""" return RedirectResult( - HTTPMethod.PUT, f"https://myarchive.example///{checksum_iri}", + HTTPMethod.PUT, ) # shared static types @@ -112,33 +115,45 @@ def url_for_put(self, checksum_iri) -> RedirectResult: # a specific implementation of some of those shared operations cls._expected_get_imp = AddonOperationImp( addon_imp=cls._my_imp, - operation=cls._expected_get_op, + declaration=cls._expected_get_op, ) cls._expected_put_imp = AddonOperationImp( addon_imp=cls._my_imp, - operation=cls._expected_put_op, + declaration=cls._expected_put_op, ) - def test_get_operations(self): + def test_get_operations(self) -> None: _protocol_dec = addon_protocol.get_declaration(self._MyProtocol) self.assertEqual( - set(_protocol_dec.get_operations()), + set(_protocol_dec.get_operation_declarations()), {self._expected_get_op, self._expected_put_op, self._expected_query_op}, ) self.assertEqual( - set(_protocol_dec.get_operations(capabilities=[self._MyCapability.GET_IT])), + set( + _protocol_dec.get_operation_declarations( + capabilities=[self._MyCapability.GET_IT] + ) + ), {self._expected_get_op, self._expected_query_op}, ) self.assertEqual( - set(_protocol_dec.get_operations(capabilities=[self._MyCapability.PUT_IT])), + set( + _protocol_dec.get_operation_declarations( + capabilities=[self._MyCapability.PUT_IT] + ) + ), {self._expected_put_op}, ) self.assertEqual( - set(_protocol_dec.get_operations(capabilities=[self._MyCapability.UNUSED])), + set( + _protocol_dec.get_operation_declarations( + capabilities=[self._MyCapability.UNUSED] + ) + ), set(), ) - def test_get_operation_imps(self): + def test_get_operation_imps(self) -> None: self.assertEqual( set(self._my_imp.get_operation_imps()), {self._expected_get_imp, self._expected_put_imp}, @@ -167,3 +182,26 @@ def test_get_operation_imps(self): ), set(), ) + + def test_operation_imp_by_name(self) -> None: + self.assertEqual( + self._my_imp.get_operation_imp_by_name("url_for_get"), + self._expected_get_imp, + ) + self.assertEqual( + self._my_imp.get_operation_imp_by_name("url_for_put"), + self._expected_put_imp, + ) + + def test_operation_call(self) -> None: + _mock_addon_instance = Mock() + _mock_addon_instance.url_for_get.return_value = RedirectResult( + "https://myarchive.example///...", + HTTPMethod.GET, + ) + _imp_for_get = self._my_imp.get_operation_imp_by_name("url_for_get") + _imp_for_get.call_with_json_kwargs( + _mock_addon_instance, + {"checksum_iri": "..."}, + ) + _mock_addon_instance.url_for_get.assert_called_once_with(checksum_iri="...") diff --git a/addon_toolkit/tests/test_constrained_http.py b/addon_toolkit/tests/test_constrained_http.py new file mode 100644 index 00000000..e2cc8648 --- /dev/null +++ b/addon_toolkit/tests/test_constrained_http.py @@ -0,0 +1,5 @@ +import addon_toolkit.constrained_http +from addon_toolkit.tests._doctest import load_doctests + + +load_tests = load_doctests(addon_toolkit.constrained_http) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 948d041f..b703c19a 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -6,4 +6,4 @@ django-filter httpx==0.26.0 celery jsonschema==4.21.1 - +aiohttp==3.9.3