From c4695053c0e1328d418129ad5f5be1b4d240ee83 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 4 Mar 2024 00:22:14 +0100 Subject: [PATCH 01/13] improve datastructures --- backoffice/_backoffice.py | 17 +- backoffice/backup.py | 2 +- backoffice/{utils/_gh.py => gh_utils.py} | 0 backoffice/remote_resource.py | 309 ++++++++++++++++++ backoffice/run_dynamic_tests.py | 8 +- backoffice/{utils => }/s3_client.py | 39 ++- .../{utils => s3_structure}/__init__.py | 0 backoffice/s3_structure/chat.py | 24 ++ backoffice/s3_structure/common.py | 13 + backoffice/s3_structure/log.py | 34 ++ backoffice/s3_structure/versions.py | 133 ++++++++ backoffice/utils/remote_resource.py | 304 ----------------- backoffice/utils/s3_structure.py | 59 ---- backoffice/validate_format.py | 17 +- setup.py | 1 + tests/conftest.py | 2 +- tests/test_utils/test_remote_resource.py | 4 +- tests/test_utils/test_s3client.py | 2 +- 18 files changed, 566 insertions(+), 402 deletions(-) rename backoffice/{utils/_gh.py => gh_utils.py} (100%) create mode 100644 backoffice/remote_resource.py rename backoffice/{utils => }/s3_client.py (88%) rename backoffice/{utils => s3_structure}/__init__.py (100%) create mode 100644 backoffice/s3_structure/chat.py create mode 100644 backoffice/s3_structure/common.py create mode 100644 backoffice/s3_structure/log.py create mode 100644 backoffice/s3_structure/versions.py delete mode 100644 backoffice/utils/remote_resource.py delete mode 100644 backoffice/utils/s3_structure.py diff --git a/backoffice/_backoffice.py b/backoffice/_backoffice.py index 91bfeec5..bd2c25f8 100644 --- a/backoffice/_backoffice.py +++ b/backoffice/_backoffice.py @@ -5,13 +5,14 @@ from dotenv import load_dotenv from backoffice.backup import backup -from backoffice.run_dynamic_tests import run_dynamic_tests -from backoffice.utils.remote_resource import ( +from backoffice.remote_resource import ( PublishedVersion, RemoteResource, StagedVersion, ) -from backoffice.utils.s3_client import Client +from backoffice.run_dynamic_tests import run_dynamic_tests +from backoffice.s3_client import Client +from backoffice.s3_structure.versions import StageNr from backoffice.validate_format import validate_format _ = load_dotenv() @@ -44,7 +45,7 @@ def stage(self, resource_id: str, package_url: str): def test( self, resource_id: str, - stage_nr: int, + stage_nr: StageNr, weight_format: Optional[Union[WeightsFormat, Literal[""]]] = None, create_env_outcome: Literal["success", ""] = "success", ): @@ -55,11 +56,15 @@ def test( create_env_outcome=create_env_outcome, ) - def await_review(self, resource_id: str, stage_nr: int): + def await_review(self, resource_id: str, stage_nr: StageNr): staged = StagedVersion(self.client, resource_id, stage_nr) staged.await_review() - def publish(self, resource_id: str, stage_nr: int): + def request_changes(self, resource_id: str, stage_nr: StageNr, reason: str): + staged = StagedVersion(self.client, resource_id, stage_nr) + staged.request_changes(reason=reason) + + def publish(self, resource_id: str, stage_nr: StageNr): staged = StagedVersion(self.client, resource_id, stage_nr) published = staged.publish() assert isinstance(published, PublishedVersion) diff --git a/backoffice/backup.py b/backoffice/backup.py index 962cce1e..b4610b4f 100644 --- a/backoffice/backup.py +++ b/backoffice/backup.py @@ -1,7 +1,7 @@ from dotenv import load_dotenv from loguru import logger -from backoffice.utils.s3_client import Client +from backoffice.s3_client import Client _ = load_dotenv() diff --git a/backoffice/utils/_gh.py b/backoffice/gh_utils.py similarity index 100% rename from backoffice/utils/_gh.py rename to backoffice/gh_utils.py diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py new file mode 100644 index 00000000..e155c269 --- /dev/null +++ b/backoffice/remote_resource.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import io +import urllib.request +import zipfile +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Generic, Optional, Type, TypeVar + +from loguru import logger +from ruyaml import YAML +from typing_extensions import assert_never + +from backoffice.s3_structure.chat import Chat +from backoffice.s3_structure.log import Logs + +from .s3_client import Client +from .s3_structure.versions import ( + AcceptedStatus, + AwaitingReviewStatus, + ChangesRequestedStatus, + PublishedStagedStatus, + PublishedStatus, + PublishedVersionDetails, + PublishNr, + StagedVersionDetails, + StagedVersionStatus, + StageNr, + SupersededStatus, + TestingStatus, + UnpackedStatus, + UnpackingStatus, + Versions, +) + +yaml = YAML(typ="safe") + +J = TypeVar("J", Versions, Logs, Chat) + +Nr = TypeVar("Nr", StageNr, PublishNr) + + +@dataclass +class RemoteResource: + """A representation of a bioimage.io resource + (**not** a specific staged or published version of it)""" + + client: Client + """Client to connect to remote storage""" + id: str + """resource identifier""" + + @property + def folder(self) -> str: + """The S3 (sub)prefix of this resource""" + return self.id + + @property + def versions_path(self) -> str: + return f"{self.id}/versions.json" + + def get_versions(self) -> Versions: + return self._get_json(Versions) + + def get_latest_stage_nr(self) -> Optional[StageNr]: + versions = self.get_versions() + if not versions.staged: + return None + else: + return max(versions.staged) + + def get_latest_staged_version(self) -> Optional[StagedVersion]: + """Get a representation of the latest staged version + (the one with the highest stage nr)""" + v = self.get_latest_stage_nr() + if v is None: + return None + else: + return StagedVersion(client=self.client, id=self.id, nr=v) + + def stage_new_version(self, package_url: str) -> StagedVersion: + """Stage the content at `package_url` as a new resource version candidate.""" + nr = self.get_latest_stage_nr() + if nr is None: + nr = StageNr(1) + + ret = StagedVersion(client=self.client, id=self.id, nr=nr) + ret.unpack(package_url=package_url) + return ret + + def _get_json(self, typ: Type[J]) -> J: + path = f"{self.folder}{type.__name__.lower()}.json" + data = self.client.load_file(path) + if data is None: + return typ() + else: + return typ.model_validate_json(data) + + def _extend_json( + self, + extension: J, + ): + path = f"{self.folder}{extension.__class__.__name__.lower()}.json" + logger.info("Extending {} with {}", path, extension) + current = self._get_json(extension.__class__) + _ = current.extend(extension) + self.client.put_pydantic(path, current) + + +@dataclass +class RemoteResourceVersion(RemoteResource, Generic[Nr], ABC): + """Base class for a resource version (`StagedVersion` or `PublishedVersion`)""" + + nr: Nr + """version number""" + + @property + @abstractmethod + def version_prefix(self) -> str: + """a prefix to distinguish independent staged and published `version` numbers""" + pass + + @property + def folder(self) -> str: + """The S3 (sub)prefix of this version + (**sub**)prefix, because the client may prefix this prefix""" + return f"{self.id}/{self.version_prefix}{self.nr}/" + + @property + def rdf_url(self) -> str: + """rdf.yaml download URL""" + return self.client.get_file_url(f"{self.folder}files/rdf.yaml") + + def get_log(self) -> Logs: + return self._get_json(Logs) + + def get_chat(self) -> Chat: + return self._get_json(Chat) + + def extend_log( + self, + extension: Logs, + ): + """extend log file""" + self._extend_json(extension) + + +@dataclass +class StagedVersion(RemoteResourceVersion[StageNr]): + """A staged resource version""" + + nr: StageNr + """stage number (**not** future resource version)""" + + @property + def version_prefix(self): + """The 'staged/' prefix identifies the `version` as a stage number + (opposed to a published resource version).""" + return "staged/" + + def unpack(self, package_url: str): + self._set_status( + UnpackingStatus(description=f"unzipping {package_url} to {self.folder}") + ) + + # Download the model zip file + try: + remotezip = urllib.request.urlopen(package_url) + except Exception: + logger.error("failed to open {}", package_url) + raise + + zipinmemory = io.BytesIO(remotezip.read()) + + # Unzip the zip file + zipobj = zipfile.ZipFile(zipinmemory) + + rdf = yaml.load(zipobj.open("rdf.yaml").read().decode()) + if (rdf_id := rdf.get("id")) is None: + rdf["id"] = self.id + elif rdf_id != self.id: + raise ValueError( + f"Expected package for {self.id}, " + f"but got packaged {rdf_id} ({package_url})" + ) + + # overwrite version information + rdf["version_nr"] = self.nr + + if rdf.get("id_emoji") is None: + # TODO: set `id_emoji` according to id + raise ValueError(f"RDF in {package_url} is missing `id_emoji`") + + for filename in zipobj.namelist(): + file_data = zipobj.open(filename).read() + path = f"{self.folder}files/{filename}" + self.client.put(path, io.BytesIO(file_data), length=len(file_data)) + + self._set_status(UnpackedStatus()) + + def set_testing_status(self, description: str): + self._set_status(TestingStatus(description=description)) + + def await_review(self): + """set status to 'awaiting review'""" + self._set_status(AwaitingReviewStatus()) + + def request_changes(self, reason: str): + self._set_status(ChangesRequestedStatus(description=reason)) + + def mark_as_superseded(self, description: str, by: StageNr): + self._set_status(SupersededStatus(description=description, by=by)) + + def publish(self) -> PublishedVersion: + """mark this staged version candidate as accepted and try to publish it""" + self._set_status(AcceptedStatus()) + versions = self.get_versions() + # check status of older staged versions + for nr, details in versions.staged.items(): + if nr >= self.nr: # ignore newer staged versions + continue + if isinstance(details.status, (SupersededStatus, PublishedStagedStatus)): + pass + elif isinstance( + details.status, + ( + UnpackingStatus, + UnpackedStatus, + TestingStatus, + AwaitingReviewStatus, + ChangesRequestedStatus, + AcceptedStatus, + ), + ): + superseded = StagedVersion(client=self.client, id=self.id, nr=nr) + superseded.mark_as_superseded(f"Superseded by {self.nr}", self.nr) + else: + assert_never(details.status) + + if not versions.published: + next_publish_nr = PublishNr(1) + else: + next_publish_nr = PublishNr(max(versions.published) + 1) + + logger.debug("Publishing {} as version nr {}", self.folder, next_publish_nr) + + # load rdf + staged_rdf_path = f"{self.folder}files/rdf.yaml" + rdf_data = self.client.load_file(staged_rdf_path) + rdf = yaml.load(rdf_data) + + sem_ver = rdf.get("version") + if sem_ver is not None and sem_ver in { + v.sem_ver for v in versions.published.values() + }: + raise RuntimeError(f"Trying to publish {sem_ver} again!") + + ret = PublishedVersion(client=self.client, id=self.id, nr=next_publish_nr) + + # copy rdf.yaml and set version in it + rdf["version_nr"] = ret.nr + stream = io.StringIO() + yaml.dump(rdf, stream) + rdf_data = stream.read().encode() + self.client.put( + f"{ret.folder}files/rdf.yaml", io.BytesIO(rdf_data), length=len(rdf_data) + ) + # self.client.rm_obj(staged_rdf_path) + + # move all other files + self.client.cp_dir(self.folder, ret.folder) + + versions.staged[self.nr].status = PublishedStagedStatus( + publish_nr=next_publish_nr + ) + versions.published[next_publish_nr] = PublishedVersionDetails( + sem_ver=sem_ver, status=PublishedStatus(stage_nr=self.nr) + ) + self._extend_json(versions) + + # TODO: clean up staged files? + # remove all uploaded files from this staged version + # self.client.rm_dir(f"{self.folder}/files/") + return ret + + def _set_status(self, value: StagedVersionStatus): + version = self.get_versions() + details = version.staged.setdefault(self.nr, StagedVersionDetails(status=value)) + if value.step < details.status.step: + logger.error("Cannot proceed from {} to {}", details.status, value) + return + + if value.step not in (details.status.step, details.status.step + 1) and not ( + details.status.name == "awaiting review" and value.name == "superseded" + ): + logger.warning("Proceeding from {} to {}", details.status, value) + + details.status = value + self._extend_json(version) + + +@dataclass +class PublishedVersion(RemoteResourceVersion[PublishNr]): + """A representation of a published resource version""" + + @property + def version_prefix(self): + """published versions do not have a prefix""" + return "" diff --git a/backoffice/run_dynamic_tests.py b/backoffice/run_dynamic_tests.py index 3956ca11..028b23d5 100644 --- a/backoffice/run_dynamic_tests.py +++ b/backoffice/run_dynamic_tests.py @@ -14,7 +14,8 @@ ) from ruyaml import YAML -from backoffice.utils.remote_resource import StagedVersion +from backoffice.remote_resource import StagedVersion +from backoffice.s3_structure.log import BioimageioLog, Logs try: from tqdm import tqdm @@ -47,8 +48,7 @@ def run_dynamic_tests( weight_format: Optional[WeightsFormat], # "weight format to test model with." create_env_outcome: str, ): - staged.set_status( - "testing", + staged.set_testing_status( "Testing" + ("" if weight_format is None else f" {weight_format} weights"), ) rdf_source = staged.rdf_url @@ -118,4 +118,4 @@ def run_dynamic_tests( ) ) - staged.add_log_entry("bioimageio.core", summary.model_dump(mode="json")) + staged.extend_log(Logs(bioimageio_core=[BioimageioLog(log=summary)])) diff --git a/backoffice/utils/s3_client.py b/backoffice/s3_client.py similarity index 88% rename from backoffice/utils/s3_client.py rename to backoffice/s3_client.py index e29ff501..e93dd89b 100644 --- a/backoffice/utils/s3_client.py +++ b/backoffice/s3_client.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, field from datetime import timedelta from pathlib import Path -from typing import Any, BinaryIO, Iterator, Optional, Union +from typing import Any, BinaryIO, Iterator, Optional, TypeVar, Union from dotenv import load_dotenv from loguru import logger @@ -15,10 +15,14 @@ from minio.commonconfig import CopySource from minio.datatypes import Object from minio.deleteobjects import DeleteObject +from pydantic import BaseModel _ = load_dotenv() +M = TypeVar("M", bound=BaseModel) + + @dataclass class Client: """Convenience wrapper around a `Minio` S3 client""" @@ -72,15 +76,19 @@ def put( ) logger.info("Uploaded {}", self.get_file_url(path)) + def put_pydantic(self, path: str, obj: BaseModel): + """convenience method to upload a json file from a pydantic model""" + self.put_json_string(path, obj.model_dump_json()) + logger.debug("Uploaded {} containing {}", self.get_file_url(path), obj) + def put_json(self, path: str, json_value: Any): """convenience method to upload a json file from a json serializable value""" - data_str = json.dumps(json_value) - data = data_str.encode() + self.put_json_string(path, json.dumps(json_value)) + logger.debug("Uploaded {} containing {}", self.get_file_url(path), json_value) + + def put_json_string(self, path: str, json_str: str): + data = json_str.encode() self.put(path, io.BytesIO(data), length=len(data)) - data_log = data_str[:1000] - if len(data_log) < len(data_str): - data_log += "..." - logger.debug("Uploaded {}", data_log) def get_file_urls( self, @@ -128,16 +136,12 @@ def ls( yield Path(obj.object_name).name + def cp_dir(self, src: str, tgt: str): + _ = self._cp_dir(src, tgt) + def mv_dir(self, src: str, tgt: str, *, bypass_governance_mode: bool = False): """copy and delete all objects under `src` to `tgt`""" - assert src.endswith("/") - assert tgt.endswith("/") - objects = list( - self._client.list_objects( - self.bucket, f"{self.prefix}/{src}", recursive=True - ) - ) - self._cp_objs(objects, src, tgt) + objects = self._cp_dir(src, tgt) self._rm_objs(objects, bypass_governance_mode=bypass_governance_mode) def rm_dir(self, prefix: str, *, bypass_governance_mode: bool = False): @@ -150,11 +154,12 @@ def rm_dir(self, prefix: str, *, bypass_governance_mode: bool = False): ) self._rm_objs(objects, bypass_governance_mode=bypass_governance_mode) - def _cp_objs(self, objects: Sequence[Object], src: str, tgt: str) -> None: + def _cp_dir(self, src: str, tgt: str): assert src.endswith("/") assert tgt.endswith("/") src = f"{self.prefix}/{src}" tgt = f"{self.prefix}/{tgt}" + objects = list(self._client.list_objects(self.bucket, src, recursive=True)) # copy for obj in objects: assert obj.object_name is not None and obj.object_name.startswith(src) @@ -166,6 +171,8 @@ def _cp_objs(self, objects: Sequence[Object], src: str, tgt: str) -> None: CopySource(self.bucket, obj.object_name), ) + return objects + def rm_obj(self, name: str) -> None: """remove single object""" self._client.remove_object(self.bucket, name) diff --git a/backoffice/utils/__init__.py b/backoffice/s3_structure/__init__.py similarity index 100% rename from backoffice/utils/__init__.py rename to backoffice/s3_structure/__init__.py diff --git a/backoffice/s3_structure/chat.py b/backoffice/s3_structure/chat.py new file mode 100644 index 00000000..dd1a541c --- /dev/null +++ b/backoffice/s3_structure/chat.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from datetime import datetime + +from pydantic import Field + +from backoffice.s3_structure.common import Node + + +class Message(Node): + author: str + text: str + timestamp: datetime = datetime.now() + + +class Chat(Node): + """`//chat.json` keeps a record of version specific comments""" + + messages: list[Message] = Field(default_factory=list) + """messages""" + + def extend(self, other: Chat): + assert set(self.model_fields) == {"messages"}, set(self.model_fields) + self.messages.extend(other.messages) diff --git a/backoffice/s3_structure/common.py b/backoffice/s3_structure/common.py new file mode 100644 index 00000000..b7a147ed --- /dev/null +++ b/backoffice/s3_structure/common.py @@ -0,0 +1,13 @@ +import pydantic + + +class Node( + pydantic.BaseModel, + extra="ignore", + frozen=False, + populate_by_name=True, + revalidate_instances="never", + validate_assignment=True, + validate_default=False, +): + pass diff --git a/backoffice/s3_structure/log.py b/backoffice/s3_structure/log.py new file mode 100644 index 00000000..2fbb9b50 --- /dev/null +++ b/backoffice/s3_structure/log.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from bioimageio.spec import ValidationSummary +from pydantic import Field + +from backoffice.s3_structure.common import Node + + +class _LogEntryBase(Node): + timestamp: datetime = datetime.now() + """creation of log entry""" + log: Any + """log content""" + + +class BioimageioLog(_LogEntryBase): + log: ValidationSummary + + +class Logs(Node): + """`//log.json` contains a version specific log""" + + bioimageio_spec: list[BioimageioLog] = Field(default_factory=list) + bioimageio_core: list[BioimageioLog] = Field(default_factory=list) + + def extend(self, other: Logs): + for k, v in other: + assert isinstance(v, list) + logs = getattr(self, k) + assert isinstance(logs, list) + logs.extend(v) diff --git a/backoffice/s3_structure/versions.py b/backoffice/s3_structure/versions.py new file mode 100644 index 00000000..d362f554 --- /dev/null +++ b/backoffice/s3_structure/versions.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Annotated, Literal, NewType, Optional, Union + +import pydantic + +from backoffice.s3_structure.common import Node + +PublishNr = NewType("PublishNr", int) +"""n-th published version""" + +StageNr = NewType("StageNr", int) +"""n-th staged version""" + + +class _StatusBase(Node): + timestamp: datetime = datetime.now() + + +class _StagedStatusBase(_StatusBase): + description: str + num_steps: Literal[6] = 6 + + @pydantic.model_validator(mode="after") + def _validate_num_steps(self): + assert self.num_steps >= getattr(self, "step", 0) + return self + + +class UnpackingStatus(_StagedStatusBase): + name: Literal["unpacking"] = "unpacking" + step: Literal[1] = 1 + + +class UnpackedStatus(_StagedStatusBase): + name: Literal["unpacked"] = "unpacked" + description: str = "staging was successful; awaiting automated tests to start ⏳" + step: Literal[2] = 2 + + +class TestingStatus(_StagedStatusBase): + name: Literal["testing"] = "testing" + step: Literal[3] = 3 + + +class AwaitingReviewStatus(_StagedStatusBase): + name: Literal["awaiting review"] = "awaiting review" + description: str = ( + "Thank you for your contribution! 🎉" + "Our bioimage.io maintainers will take a look soon. 🦒" + ) + + step: Literal[4] = 4 + + +class ChangesRequestedStatus(_StagedStatusBase): + name: Literal["changes requested"] = "changes requested" + step: Literal[5] = 5 + + +class AcceptedStatus(_StagedStatusBase): + name: Literal["accepted"] = "accepted" + description: str = ( + "This staged version has been accepted by a bioimage.io maintainer and is about to be published." + ) + step: Literal[5] = 5 + + +class SupersededStatus(_StagedStatusBase): + """following `ChangesRequestedStatus` and staging of a superseding staged version""" + + name: Literal["superseded"] = "superseded" + step: Literal[6] = 6 + by: StageNr + + +class PublishedStagedStatus(_StagedStatusBase): + """following `AcceptedStatus`""" + + name: Literal["published"] = "published" + description: str = "published! 🎉" + step: Literal[6] = 6 + publish_nr: PublishNr + + +StagedVersionStatus = Annotated[ + Union[ + UnpackingStatus, + UnpackedStatus, + TestingStatus, + AwaitingReviewStatus, + ChangesRequestedStatus, + AcceptedStatus, + SupersededStatus, + PublishedStagedStatus, + ], + pydantic.Discriminator("name"), +] + + +class PublishedStatus(_StatusBase): + name: Literal["published"] = "published" + stage_nr: StageNr + + +PulishedVersionStatus = PublishedStatus + + +class VersionDetails(Node): + sem_ver: Optional[str] = None + + +class PublishedVersionDetails(VersionDetails): + status: PublishedStatus + + +class StagedVersionDetails(VersionDetails): + status: StagedVersionStatus + + +class Versions(Node): + """`/versions.json` containing an overview of all published and staged resource versions""" + + published: dict[PublishNr, PublishedVersionDetails] = pydantic.Field( + default_factory=dict + ) + staged: dict[StageNr, StagedVersionDetails] = pydantic.Field(default_factory=dict) + + def extend(self, other: Versions) -> None: + assert set(self.model_fields) == {"published", "staged"}, set(self.model_fields) + self.published.update(other.published) + self.staged.update(other.staged) diff --git a/backoffice/utils/remote_resource.py b/backoffice/utils/remote_resource.py deleted file mode 100644 index cd162989..00000000 --- a/backoffice/utils/remote_resource.py +++ /dev/null @@ -1,304 +0,0 @@ -from __future__ import annotations - -import io -import json -import urllib.request -import zipfile -from abc import ABC, abstractmethod -from dataclasses import dataclass -from datetime import datetime -from typing import Any, Optional, Union - -from loguru import logger -from ruyaml import YAML -from typing_extensions import assert_never - -from .s3_client import Client -from .s3_structure import ( - Details, - Log, - LogCategory, - Status, - StatusName, - VersionDetails, - Versions, -) - -yaml = YAML(typ="safe") - - -@dataclass -class RemoteResource: - """A representation of a bioimage.io resource - (**not** a specific staged or published version of it)""" - - client: Client - """Client to connect to remote storage""" - id: str - """resource identifier""" - - @property - def versions_path(self) -> str: - return f"{self.id}/versions.json" - - def get_published_versions(self) -> Versions: - versions_data = self.client.load_file(self.versions_path) - if versions_data is None: - versions: Versions = {} - else: - versions = json.loads(versions_data) - assert isinstance(versions, dict) - return versions - - def _get_latest_stage_nr(self) -> Optional[int]: - staged = list(map(int, self.client.ls(f"{self.id}/staged/", only_folders=True))) - if not staged: - return None - else: - return max(staged) - - def get_latest_staged_version(self) -> Optional[StagedVersion]: - """Get a representation of the latest staged version - (the one with the highest stage nr)""" - v = self._get_latest_stage_nr() - if v is None: - return None - else: - return StagedVersion(client=self.client, id=self.id, version=v) - - def stage_new_version(self, package_url: str) -> StagedVersion: - """Stage the content at `package_url` as a new resource version candidate.""" - v = self._get_latest_stage_nr() - if v is None: - v = 1 - - ret = StagedVersion(client=self.client, id=self.id, version=v) - ret.set_status("staging", f"unzipping {package_url} to {ret.folder}") - - # Download the model zip file - try: - remotezip = urllib.request.urlopen(package_url) - except Exception: - logger.error("failed to open {}", package_url) - raise - - zipinmemory = io.BytesIO(remotezip.read()) - - # Unzip the zip file - zipobj = zipfile.ZipFile(zipinmemory) - - rdf = yaml.load(zipobj.open("rdf.yaml").read().decode()) - if (rdf_id := rdf.get("id")) is None: - rdf["id"] = ret.id - elif rdf_id != ret.id: - raise ValueError( - f"Expected package for {ret.id}, " - f"but got packaged {rdf_id} ({package_url})" - ) - - # overwrite version information - rdf["version"] = ret.version - - if rdf.get("id_emoji") is None: - # TODO: set `id_emoji` according to id - raise ValueError(f"RDF in {package_url} is missing `id_emoji`") - - for filename in zipobj.namelist(): - file_data = zipobj.open(filename).read() - path = f"{ret.folder}files/{filename}" - self.client.put(path, io.BytesIO(file_data), length=len(file_data)) - - return ret - - -@dataclass -class RemoteResourceVersion(RemoteResource, ABC): - """Base class for a resource version (`StagedVersion` or `PublishedVersion`)""" - - version: int - """version number""" - - @property - @abstractmethod - def version_prefix(self) -> str: - """a prefix to distinguish independent staged and published `version` numbers""" - pass - - @property - def folder(self) -> str: - """The S3 (sub)prefix of this version - (**sub**)prefix, because the client may prefix this prefix""" - return f"{self.id}/{self.version_prefix}{self.version}/" - - @property - def rdf_url(self) -> str: - """rdf.yaml download URL""" - return self.client.get_file_url(f"{self.folder}files/rdf.yaml") - - def get_log(self) -> Log: - path = f"{self.folder}log.json" - log_data = self.client.load_file(path) - if log_data is None: - log: Log = {} - else: - log = json.loads(log_data) - assert isinstance(log, dict) - - return log - - def _get_details(self) -> Details: - details_data = self.client.load_file(f"{self.folder}details.json") - if details_data is None: - details: Details = { - "messages": [], - "status": self._create_status("unknown", "no status information found"), - } - else: - details = json.load(io.BytesIO(details_data)) - - return details - - def _set_details(self, details: Details): - self.client.put_json(f"{self.folder}details.json", details) - - def get_messages(self): - details = self._get_details() - return details["messages"] - - def add_message(self, author: str, text: str): - logger.info("msg from {}: text", author) - details = self._get_details() - now = datetime.now().isoformat() - details["messages"].append({"author": author, "text": text, "time": now}) - self._set_details(details) - - def set_status(self, name: StatusName, description: str) -> None: - details = self._get_details() - details["status"] = self._create_status(name, description) - self._set_details(details) - - @staticmethod - def _create_status(name: StatusName, description: str) -> Status: - num_steps = 5 - if name == "unknown": - step = 1 - num_steps = 1 - elif name == "staging": - step = 1 - elif name == "testing": - step = 2 - elif name == "awaiting review": - step = 3 - elif name == "publishing": - step = 4 - elif name == "published": - step = 5 - else: - assert_never(name) - - return Status( - name=name, description=description, step=step, num_steps=num_steps - ) - - def get_status(self) -> Status: - """get the current status""" - details = self._get_details() - return details["status"] - - def add_log_entry( - self, - category: LogCategory, - content: Union[list[Any], dict[Any, Any], int, float, str, None, bool], - ): - """add log entry""" - log = self.get_log() - entries = log.setdefault(category, []) - now = datetime.now().isoformat() - entries.append({"timestamp": now, "log": content}) - self._set_log(log) - - def _set_log(self, log: Log) -> None: - self.client.put_json(f"{self.folder}log.json", log) - - -@dataclass -class StagedVersion(RemoteResourceVersion): - """A staged resource version""" - - version: int - """stage number (**not** future resource version)""" - - @property - def version_prefix(self): - """The 'staged/' prefix identifies the `version` as a stage number - (opposed to a published resource version).""" - return "staged/" - - def await_review(self): - """set status to 'awaiting review'""" - self.set_status( - "awaiting review", - description=( - "Thank you for your contribution! " - "Our bioimage.io maintainers will take a look soon." - ), - ) - - def publish(self) -> PublishedVersion: - """publish this staged version candidate as the next resource version""" - # get next version and update versions.json - versions = self.get_published_versions() - if not versions: - next_version = 1 - else: - next_version = max(map(int, versions)) + 1 - - logger.debug("Publishing {} as version nr {}", self.folder, next_version) - - assert next_version not in versions, (next_version, versions) - - # load rdf - staged_rdf_path = f"{self.folder}files/rdf.yaml" - rdf_data = self.client.load_file(staged_rdf_path) - rdf = yaml.load(rdf_data) - - sem_ver = rdf.get("version") - if sem_ver is not None and sem_ver in {v["sem_ver"] for v in versions.values()}: - raise RuntimeError(f"Trying to publish {sem_ver} again!") - - versions[next_version] = VersionDetails(sem_ver=sem_ver) - - updated_versions_data = json.dumps(versions).encode() - self.client.put( - self.versions_path, - io.BytesIO(updated_versions_data), - length=len(updated_versions_data), - ) - ret = PublishedVersion(client=self.client, id=self.id, version=next_version) - - # move rdf.yaml and set version in it - rdf["version"] = ret.version - stream = io.StringIO() - yaml.dump(rdf, stream) - rdf_data = stream.read().encode() - self.client.put( - f"{ret.folder}files/rdf.yaml", io.BytesIO(rdf_data), length=len(rdf_data) - ) - self.client.rm_obj(staged_rdf_path) - - # move all other files - self.client.mv_dir(self.folder, ret.folder) - - # remove all preceding staged versions - self.client.rm_dir(f"{self.id}/{self.version_prefix}") - return ret - - -@dataclass -class PublishedVersion(RemoteResourceVersion): - """A representation of a published resource version""" - - @property - def version_prefix(self): - """published versions do not have a prefix""" - return "" diff --git a/backoffice/utils/s3_structure.py b/backoffice/utils/s3_structure.py deleted file mode 100644 index 358c22fd..00000000 --- a/backoffice/utils/s3_structure.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Descriptions of -- `/versions.json` `Versions` -- `//log.json` → `Log` -- `//details.json` → `Details` -""" - -from typing import Any, Literal, Optional, TypedDict - - -class VersionDetails(TypedDict): - sem_ver: Optional[str] - - -VersionNr = int -"""the n-th published version""" - -Versions = dict[VersionNr, VersionDetails] -"""info about published resource versions at `/versions.json`""" - -LogCategory = Literal[ - "bioimageio.spec", "bioimageio.core", "ilastik", "deepimagej", "icy", "biapy" -] - - -class LogEntry(TypedDict): - timestamp: str - """creation of log entry""" - log: Any - """log content""" - - -Log = dict[LogCategory, list[LogEntry]] -"""version specific log at `//log.json`""" - - -class Message(TypedDict): - author: str - text: str - time: str - """time in ISO 8601""" - - -StatusName = Literal["unknown", "staging", "testing", "awaiting review"] - - -class Status(TypedDict): - name: StatusName - description: str - step: int - num_steps: int - - -class Details(TypedDict): - """version specific details at `//details.json`""" - - messages: list[Message] - """messages""" - status: Status diff --git a/backoffice/validate_format.py b/backoffice/validate_format.py index 92758497..393127ac 100644 --- a/backoffice/validate_format.py +++ b/backoffice/validate_format.py @@ -11,8 +11,9 @@ from ruyaml import YAML from typing_extensions import assert_never -from backoffice.utils._gh import set_multiple_gh_actions_outputs -from backoffice.utils.remote_resource import StagedVersion +from backoffice.gh_utils import set_multiple_gh_actions_outputs +from backoffice.remote_resource import StagedVersion +from backoffice.s3_structure.log import BioimageioLog, Logs yaml = YAML(typ="safe") @@ -219,7 +220,7 @@ def prepare_dynamic_test_cases( def validate_format(staged: StagedVersion): - staged.set_status("testing", "Testing RDF format") + staged.set_testing_status("Validating RDF format") rdf_source = staged.rdf_url rd = load_description(rdf_source, format_version="discover") dynamic_test_cases: list[dict[Literal["weight_format"], WeightsFormat]] = [] @@ -234,8 +235,8 @@ def validate_format(staged: StagedVersion): rd = rd_latest rd.validation_summary.status = "passed" # passed in 'discover' mode if not isinstance(rd, InvalidDescr) and rd.version is not None: - published = staged.get_published_versions() - if str(rd.version) in {v["sem_ver"] for v in published.values()}: + published = staged.get_versions().published + if str(rd.version) in {v.sem_ver for v in published.values()}: error = ErrorEntry( loc=("version",), msg=f"Trying to publish version {rd.version} again!", @@ -252,14 +253,14 @@ def validate_format(staged: StagedVersion): ) ) - summary = rd.validation_summary.model_dump(mode="json") - staged.add_log_entry("bioimageio.spec", summary) + summary = rd.validation_summary + staged.extend_log(Logs(bioimageio_spec=[BioimageioLog(log=summary)])) set_multiple_gh_actions_outputs( dict( has_dynamic_test_cases=bool(dynamic_test_cases), dynamic_test_cases={"include": dynamic_test_cases}, - version=staged.version, + version=staged.nr, conda_envs=conda_envs, ) ) diff --git a/setup.py b/setup.py index 9355b89a..ffd90b20 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ "loguru", "minio==7.2.4", "ruyaml", + "pydantic==2.6.3", "tqdm", ], extras_require={ diff --git a/tests/conftest.py b/tests/conftest.py index fd85f189..93b24ec5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import pytest from backoffice import BackOffice -from backoffice.utils.s3_client import Client +from backoffice.s3_client import Client @pytest.fixture(scope="session") diff --git a/tests/test_utils/test_remote_resource.py b/tests/test_utils/test_remote_resource.py index 791395dc..187718bf 100644 --- a/tests/test_utils/test_remote_resource.py +++ b/tests/test_utils/test_remote_resource.py @@ -1,12 +1,12 @@ import os from backoffice.backup import backup -from backoffice.utils.remote_resource import ( +from backoffice.remote_resource import ( PublishedVersion, RemoteResource, StagedVersion, ) -from backoffice.utils.s3_client import Client +from backoffice.s3_client import Client def test_lifecycle( diff --git a/tests/test_utils/test_s3client.py b/tests/test_utils/test_s3client.py index 9776212e..5b408411 100644 --- a/tests/test_utils/test_s3client.py +++ b/tests/test_utils/test_s3client.py @@ -1,4 +1,4 @@ -from backoffice.utils.s3_client import Client +from backoffice.s3_client import Client def test_client(client: Client): From e0370f207012ad29703c35f904d5ef58068dde74 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 4 Mar 2024 00:23:45 +0100 Subject: [PATCH 02/13] add request_changes wfs --- .github/workflows/request_changes.yaml | 41 ++++++++++++++++ .github/workflows/request_changes_call.yaml | 52 +++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 .github/workflows/request_changes.yaml create mode 100644 .github/workflows/request_changes_call.yaml diff --git a/.github/workflows/request_changes.yaml b/.github/workflows/request_changes.yaml new file mode 100644 index 00000000..ec5c4a59 --- /dev/null +++ b/.github/workflows/request_changes.yaml @@ -0,0 +1,41 @@ +name: request changes + +on: + workflow_dispatch: + inputs: + resource_id: + description: "Bioimageio ID of the resource - to be used to access the resource on S3" + required: true + type: string + stage_nr: + description: stage nr + required: true + type: number + reason: + description: Why are changes required? What needs changing? + required: true + type: string + +concurrency: ${{inputs.resource_id}} + +env: + S3_HOST: ${{vars.S3_HOST}} + S3_BUCKET: ${{vars.S3_BUCKET}} + S3_FOLDER: ${{vars.S3_FOLDER}} + S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}} + S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} + ZENODO_URL: ${{vars.ZENODO_URL}} + ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}} + +jobs: + call: + uses: ./.github/workflows/request_changes_call.yaml + with: + resource_id: ${{inputs.resource_id}} + stage_nr: ${{inputs.stage_nr}} + reason: ${{inputs.reason}} + S3_HOST: ${{vars.S3_HOST}} + S3_BUCKET: ${{vars.S3_BUCKET}} + S3_FOLDER: ${{vars.S3_FOLDER}} + ZENODO_URL: ${{vars.ZENODO_URL}} + secrets: inherit diff --git a/.github/workflows/request_changes_call.yaml b/.github/workflows/request_changes_call.yaml new file mode 100644 index 00000000..8b3670c5 --- /dev/null +++ b/.github/workflows/request_changes_call.yaml @@ -0,0 +1,52 @@ +name: request changes call + +on: + workflow_call: + inputs: + resource_id: + description: "Bioimageio ID of the resource - to be used to access the resource on S3" + required: true + type: string + stage_nr: + description: stage nr + required: true + type: number + reason: + description: Why are changes required? What needs changing? + required: true + type: string + S3_HOST: + required: true + type: string + S3_BUCKET: + required: true + type: string + S3_FOLDER: + required: true + type: string + ZENODO_URL: + required: true + type: string + +concurrency: ${{inputs.resource_id}}-call + +env: + S3_HOST: ${{inputs.S3_HOST}} + S3_BUCKET: ${{inputs.S3_BUCKET}} + S3_FOLDER: ${{inputs.S3_FOLDER}} + ZENODO_URL: ${{inputs.ZENODO_URL}} + S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}} + S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} + ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}} + +jobs: + request-changes: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" # caching pip dependencies + - run: pip install . + - run: backoffice request-changes "${{ inputs.resource_id }}" "${{ inputs.stage_nr }}" "${{ inputs.reason }}" From 3b9e6f232c78c095d81e3f68336036bd37f489ad Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 4 Mar 2024 00:38:21 +0100 Subject: [PATCH 03/13] fix pyright check --- tests/test_backoffice.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_backoffice.py b/tests/test_backoffice.py index 8b3c196d..49fc809f 100644 --- a/tests/test_backoffice.py +++ b/tests/test_backoffice.py @@ -1,13 +1,14 @@ import os from backoffice import BackOffice +from backoffice.s3_structure.versions import StageNr def test_backoffice( backoffice: BackOffice, package_url: str, package_id: str, s3_test_folder_url: str ): backoffice.stage(resource_id=package_id, package_url=package_url) - backoffice.test(resource_id=package_id, stage_nr=1) - backoffice.await_review(resource_id=package_id, stage_nr=1) - backoffice.publish(resource_id=package_id, stage_nr=1) + backoffice.test(resource_id=package_id, stage_nr=StageNr(1)) + backoffice.await_review(resource_id=package_id, stage_nr=StageNr(1)) + backoffice.publish(resource_id=package_id, stage_nr=StageNr(1)) backoffice.backup(os.environ["ZENODO_TEST_URL"]) From 8ba7b2846a082b6268b2b0c13a48a312d77be1c5 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 4 Mar 2024 00:38:41 +0100 Subject: [PATCH 04/13] version -> versions --- backoffice/remote_resource.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py index e155c269..fc2ee688 100644 --- a/backoffice/remote_resource.py +++ b/backoffice/remote_resource.py @@ -284,8 +284,10 @@ def publish(self) -> PublishedVersion: return ret def _set_status(self, value: StagedVersionStatus): - version = self.get_versions() - details = version.staged.setdefault(self.nr, StagedVersionDetails(status=value)) + versions = self.get_versions() + details = versions.staged.setdefault( + self.nr, StagedVersionDetails(status=value) + ) if value.step < details.status.step: logger.error("Cannot proceed from {} to {}", details.status, value) return @@ -296,7 +298,7 @@ def _set_status(self, value: StagedVersionStatus): logger.warning("Proceeding from {} to {}", details.status, value) details.status = value - self._extend_json(version) + self._extend_json(versions) @dataclass From a10dbd83065d500877c7eb104ee8345fb824e727 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 4 Mar 2024 00:41:38 +0100 Subject: [PATCH 05/13] fix _get_json --- backoffice/remote_resource.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py index fc2ee688..263d0aea 100644 --- a/backoffice/remote_resource.py +++ b/backoffice/remote_resource.py @@ -55,10 +55,6 @@ def folder(self) -> str: """The S3 (sub)prefix of this resource""" return self.id - @property - def versions_path(self) -> str: - return f"{self.id}/versions.json" - def get_versions(self) -> Versions: return self._get_json(Versions) @@ -89,7 +85,7 @@ def stage_new_version(self, package_url: str) -> StagedVersion: return ret def _get_json(self, typ: Type[J]) -> J: - path = f"{self.folder}{type.__name__.lower()}.json" + path = f"{self.folder}{typ.__name__.lower()}.json" data = self.client.load_file(path) if data is None: return typ() From 3b34d868251f6672fd03f4843f709fdaf61cf605 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 4 Mar 2024 00:47:34 +0100 Subject: [PATCH 06/13] fix sem_ver is string --- backoffice/remote_resource.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py index 263d0aea..3c09a512 100644 --- a/backoffice/remote_resource.py +++ b/backoffice/remote_resource.py @@ -246,10 +246,10 @@ def publish(self) -> PublishedVersion: rdf = yaml.load(rdf_data) sem_ver = rdf.get("version") - if sem_ver is not None and sem_ver in { - v.sem_ver for v in versions.published.values() - }: - raise RuntimeError(f"Trying to publish {sem_ver} again!") + if sem_ver is not None: + sem_ver = str(sem_ver) + if sem_ver in {v.sem_ver for v in versions.published.values()}: + raise RuntimeError(f"Trying to publish {sem_ver} again!") ret = PublishedVersion(client=self.client, id=self.id, nr=next_publish_nr) From 9332627841cc87a210a00c2293e8a144236be977 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 5 Mar 2024 11:14:09 +0100 Subject: [PATCH 07/13] make variable names more expressive nr -> number stage_nr -> stage_number publish_nr -> publish_number StageNr -> StageNumber PublishNr -> PublishNumber J -> JsonFileT Nr -> NumberT --- .github/workflows/publish.yaml | 6 +- .github/workflows/publish_call.yaml | 6 +- .github/workflows/request_changes.yaml | 6 +- .github/workflows/request_changes_call.yaml | 6 +- .github/workflows/test.yaml | 2 +- backoffice/_backoffice.py | 20 +++--- backoffice/remote_resource.py | 68 +++++++++++---------- backoffice/s3_structure/versions.py | 16 ++--- backoffice/validate_format.py | 2 +- setup.py | 2 +- tests/test_backoffice.py | 8 +-- 11 files changed, 74 insertions(+), 68 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 5c8fac2c..c4156cd0 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -7,8 +7,8 @@ on: description: "Bioimageio ID of the resource - to be used to access the resource on S3" required: true type: string - stage_nr: - description: stage nr to publish + stage_number: + description: stage number to publish required: true type: number @@ -28,7 +28,7 @@ jobs: uses: ./.github/workflows/publish_call.yaml with: resource_id: ${{inputs.resource_id}} - stage_nr: ${{inputs.stage_nr}} + stage_number: ${{inputs.stage_number}} S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} S3_FOLDER: ${{vars.S3_FOLDER}} diff --git a/.github/workflows/publish_call.yaml b/.github/workflows/publish_call.yaml index 8a890d7b..4796c9f4 100644 --- a/.github/workflows/publish_call.yaml +++ b/.github/workflows/publish_call.yaml @@ -7,8 +7,8 @@ on: description: "Bioimageio ID of the resource - to be used to access the resource on S3" required: true type: string - stage_nr: - description: stage nr to publish + stage_number: + description: stage number to publish required: true type: number S3_HOST: @@ -45,7 +45,7 @@ jobs: python-version: "3.12" cache: "pip" # caching pip dependencies - run: pip install . - - run: backoffice publish "${{ inputs.resource_id }}" "${{ inputs.stage_nr }}" + - run: backoffice publish "${{ inputs.resource_id }}" "${{ inputs.stage_number }}" # - name: Publish to Zenodo # run: | # python .github/scripts/update_status.py "${{ inputs.resource_path }}" "Publishing to Zenodo" "5" diff --git a/.github/workflows/request_changes.yaml b/.github/workflows/request_changes.yaml index ec5c4a59..60bac931 100644 --- a/.github/workflows/request_changes.yaml +++ b/.github/workflows/request_changes.yaml @@ -7,8 +7,8 @@ on: description: "Bioimageio ID of the resource - to be used to access the resource on S3" required: true type: string - stage_nr: - description: stage nr + stage_number: + description: stage number required: true type: number reason: @@ -32,7 +32,7 @@ jobs: uses: ./.github/workflows/request_changes_call.yaml with: resource_id: ${{inputs.resource_id}} - stage_nr: ${{inputs.stage_nr}} + stage_number: ${{inputs.stage_number}} reason: ${{inputs.reason}} S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} diff --git a/.github/workflows/request_changes_call.yaml b/.github/workflows/request_changes_call.yaml index 8b3670c5..a166ac2d 100644 --- a/.github/workflows/request_changes_call.yaml +++ b/.github/workflows/request_changes_call.yaml @@ -7,8 +7,8 @@ on: description: "Bioimageio ID of the resource - to be used to access the resource on S3" required: true type: string - stage_nr: - description: stage nr + stage_number: + description: stage number required: true type: number reason: @@ -49,4 +49,4 @@ jobs: python-version: "3.12" cache: "pip" # caching pip dependencies - run: pip install . - - run: backoffice request-changes "${{ inputs.resource_id }}" "${{ inputs.stage_nr }}" "${{ inputs.reason }}" + - run: backoffice request-changes "${{ inputs.resource_id }}" "${{ inputs.stage_number }}" "${{ inputs.reason }}" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f1781559..957dcd40 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -49,7 +49,7 @@ jobs: uses: ./.github/workflows/publish_call.yaml with: resource_id: ${{vars.TEST_PACKAGE_ID}} # testing! - stage_nr: 1 + stage_number: 1 S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_TEST_BUCKET}} # testing! S3_FOLDER: ${{vars.S3_TEST_FOLDER}}/ci # testing! diff --git a/backoffice/_backoffice.py b/backoffice/_backoffice.py index bd2c25f8..5e5e86b1 100644 --- a/backoffice/_backoffice.py +++ b/backoffice/_backoffice.py @@ -12,7 +12,7 @@ ) from backoffice.run_dynamic_tests import run_dynamic_tests from backoffice.s3_client import Client -from backoffice.s3_structure.versions import StageNr +from backoffice.s3_structure.versions import StageNumber from backoffice.validate_format import validate_format _ = load_dotenv() @@ -45,27 +45,29 @@ def stage(self, resource_id: str, package_url: str): def test( self, resource_id: str, - stage_nr: StageNr, + stage_number: StageNumber, weight_format: Optional[Union[WeightsFormat, Literal[""]]] = None, create_env_outcome: Literal["success", ""] = "success", ): - staged = StagedVersion(self.client, resource_id, stage_nr) + staged = StagedVersion(self.client, resource_id, stage_number) run_dynamic_tests( staged=staged, weight_format=weight_format or None, create_env_outcome=create_env_outcome, ) - def await_review(self, resource_id: str, stage_nr: StageNr): - staged = StagedVersion(self.client, resource_id, stage_nr) + def await_review(self, resource_id: str, stage_number: StageNumber): + staged = StagedVersion(self.client, resource_id, stage_number) staged.await_review() - def request_changes(self, resource_id: str, stage_nr: StageNr, reason: str): - staged = StagedVersion(self.client, resource_id, stage_nr) + def request_changes( + self, resource_id: str, stage_numbermber: StageNumber, reason: str + ): + staged = StagedVersion(self.client, resource_id, stage_number) staged.request_changes(reason=reason) - def publish(self, resource_id: str, stage_nr: StageNr): - staged = StagedVersion(self.client, resource_id, stage_nr) + def publish(self, resource_id: str, stage_number: StageNumber): + staged = StagedVersion(self.client, resource_id, stage_number) published = staged.publish() assert isinstance(published, PublishedVersion) diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py index 3c09a512..20c14b60 100644 --- a/backoffice/remote_resource.py +++ b/backoffice/remote_resource.py @@ -22,10 +22,10 @@ PublishedStagedStatus, PublishedStatus, PublishedVersionDetails, - PublishNr, + PublishNumber, StagedVersionDetails, StagedVersionStatus, - StageNr, + StageNumber, SupersededStatus, TestingStatus, UnpackedStatus, @@ -35,9 +35,9 @@ yaml = YAML(typ="safe") -J = TypeVar("J", Versions, Logs, Chat) +JsonFileT = TypeVar("JsonFileT", Versions, Logs, Chat) -Nr = TypeVar("Nr", StageNr, PublishNr) +NumberT = TypeVar("NumberT", StageNumber, PublishNumber) @dataclass @@ -58,7 +58,7 @@ def folder(self) -> str: def get_versions(self) -> Versions: return self._get_json(Versions) - def get_latest_stage_nr(self) -> Optional[StageNr]: + def get_latest_stage_number(self) -> Optional[StageNumber]: versions = self.get_versions() if not versions.staged: return None @@ -67,24 +67,24 @@ def get_latest_stage_nr(self) -> Optional[StageNr]: def get_latest_staged_version(self) -> Optional[StagedVersion]: """Get a representation of the latest staged version - (the one with the highest stage nr)""" - v = self.get_latest_stage_nr() - if v is None: + (the one with the highest stage number)""" + nr = self.get_latest_stage_number() + if nr is None: return None else: - return StagedVersion(client=self.client, id=self.id, nr=v) + return StagedVersion(client=self.client, id=self.id, number=nr) def stage_new_version(self, package_url: str) -> StagedVersion: """Stage the content at `package_url` as a new resource version candidate.""" - nr = self.get_latest_stage_nr() + nr = self.get_latest_stage_number() if nr is None: - nr = StageNr(1) + nr = StageNumber(1) - ret = StagedVersion(client=self.client, id=self.id, nr=nr) + ret = StagedVersion(client=self.client, id=self.id, number=nr) ret.unpack(package_url=package_url) return ret - def _get_json(self, typ: Type[J]) -> J: + def _get_json(self, typ: Type[JsonFileT]) -> JsonFileT: path = f"{self.folder}{typ.__name__.lower()}.json" data = self.client.load_file(path) if data is None: @@ -94,7 +94,7 @@ def _get_json(self, typ: Type[J]) -> J: def _extend_json( self, - extension: J, + extension: JsonFileT, ): path = f"{self.folder}{extension.__class__.__name__.lower()}.json" logger.info("Extending {} with {}", path, extension) @@ -104,10 +104,10 @@ def _extend_json( @dataclass -class RemoteResourceVersion(RemoteResource, Generic[Nr], ABC): +class RemoteResourceVersion(RemoteResource, Generic[NumberT], ABC): """Base class for a resource version (`StagedVersion` or `PublishedVersion`)""" - nr: Nr + number: NumberT """version number""" @property @@ -120,7 +120,7 @@ def version_prefix(self) -> str: def folder(self) -> str: """The S3 (sub)prefix of this version (**sub**)prefix, because the client may prefix this prefix""" - return f"{self.id}/{self.version_prefix}{self.nr}/" + return f"{self.id}/{self.version_prefix}{self.number}/" @property def rdf_url(self) -> str: @@ -142,10 +142,10 @@ def extend_log( @dataclass -class StagedVersion(RemoteResourceVersion[StageNr]): +class StagedVersion(RemoteResourceVersion[StageNumber]): """A staged resource version""" - nr: StageNr + number: StageNumber """stage number (**not** future resource version)""" @property @@ -181,7 +181,7 @@ def unpack(self, package_url: str): ) # overwrite version information - rdf["version_nr"] = self.nr + rdf["version_number"] = self.number if rdf.get("id_emoji") is None: # TODO: set `id_emoji` according to id @@ -204,7 +204,7 @@ def await_review(self): def request_changes(self, reason: str): self._set_status(ChangesRequestedStatus(description=reason)) - def mark_as_superseded(self, description: str, by: StageNr): + def mark_as_superseded(self, description: str, by: StageNumber): self._set_status(SupersededStatus(description=description, by=by)) def publish(self) -> PublishedVersion: @@ -213,7 +213,7 @@ def publish(self) -> PublishedVersion: versions = self.get_versions() # check status of older staged versions for nr, details in versions.staged.items(): - if nr >= self.nr: # ignore newer staged versions + if nr >= self.number: # ignore newer staged versions continue if isinstance(details.status, (SupersededStatus, PublishedStagedStatus)): pass @@ -228,15 +228,17 @@ def publish(self) -> PublishedVersion: AcceptedStatus, ), ): - superseded = StagedVersion(client=self.client, id=self.id, nr=nr) - superseded.mark_as_superseded(f"Superseded by {self.nr}", self.nr) + superseded = StagedVersion(client=self.client, id=self.id, number=nr) + superseded.mark_as_superseded( + f"Superseded by {self.number}", self.number + ) else: assert_never(details.status) if not versions.published: - next_publish_nr = PublishNr(1) + next_publish_nr = PublishNumber(1) else: - next_publish_nr = PublishNr(max(versions.published) + 1) + next_publish_nr = PublishNumber(max(versions.published) + 1) logger.debug("Publishing {} as version nr {}", self.folder, next_publish_nr) @@ -251,10 +253,10 @@ def publish(self) -> PublishedVersion: if sem_ver in {v.sem_ver for v in versions.published.values()}: raise RuntimeError(f"Trying to publish {sem_ver} again!") - ret = PublishedVersion(client=self.client, id=self.id, nr=next_publish_nr) + ret = PublishedVersion(client=self.client, id=self.id, number=next_publish_nr) # copy rdf.yaml and set version in it - rdf["version_nr"] = ret.nr + rdf["version_number"] = ret.number stream = io.StringIO() yaml.dump(rdf, stream) rdf_data = stream.read().encode() @@ -266,11 +268,11 @@ def publish(self) -> PublishedVersion: # move all other files self.client.cp_dir(self.folder, ret.folder) - versions.staged[self.nr].status = PublishedStagedStatus( - publish_nr=next_publish_nr + versions.staged[self.number].status = PublishedStagedStatus( + publish_number=next_publish_nr ) versions.published[next_publish_nr] = PublishedVersionDetails( - sem_ver=sem_ver, status=PublishedStatus(stage_nr=self.nr) + sem_ver=sem_ver, status=PublishedStatus(stage_number=self.number) ) self._extend_json(versions) @@ -282,7 +284,7 @@ def publish(self) -> PublishedVersion: def _set_status(self, value: StagedVersionStatus): versions = self.get_versions() details = versions.staged.setdefault( - self.nr, StagedVersionDetails(status=value) + self.number, StagedVersionDetails(status=value) ) if value.step < details.status.step: logger.error("Cannot proceed from {} to {}", details.status, value) @@ -298,7 +300,7 @@ def _set_status(self, value: StagedVersionStatus): @dataclass -class PublishedVersion(RemoteResourceVersion[PublishNr]): +class PublishedVersion(RemoteResourceVersion[PublishNumber]): """A representation of a published resource version""" @property diff --git a/backoffice/s3_structure/versions.py b/backoffice/s3_structure/versions.py index d362f554..55663c4f 100644 --- a/backoffice/s3_structure/versions.py +++ b/backoffice/s3_structure/versions.py @@ -7,10 +7,10 @@ from backoffice.s3_structure.common import Node -PublishNr = NewType("PublishNr", int) +PublishNumber = NewType("PublishNumber", int) """n-th published version""" -StageNr = NewType("StageNr", int) +StageNumber = NewType("StageNumber", int) """n-th staged version""" @@ -72,7 +72,7 @@ class SupersededStatus(_StagedStatusBase): name: Literal["superseded"] = "superseded" step: Literal[6] = 6 - by: StageNr + by: StageNumber class PublishedStagedStatus(_StagedStatusBase): @@ -81,7 +81,7 @@ class PublishedStagedStatus(_StagedStatusBase): name: Literal["published"] = "published" description: str = "published! 🎉" step: Literal[6] = 6 - publish_nr: PublishNr + publish_number: PublishNumber StagedVersionStatus = Annotated[ @@ -101,7 +101,7 @@ class PublishedStagedStatus(_StagedStatusBase): class PublishedStatus(_StatusBase): name: Literal["published"] = "published" - stage_nr: StageNr + stage_number: StageNumber PulishedVersionStatus = PublishedStatus @@ -122,10 +122,12 @@ class StagedVersionDetails(VersionDetails): class Versions(Node): """`/versions.json` containing an overview of all published and staged resource versions""" - published: dict[PublishNr, PublishedVersionDetails] = pydantic.Field( + published: dict[PublishNumber, PublishedVersionDetails] = pydantic.Field( + default_factory=dict + ) + staged: dict[StageNumber, StagedVersionDetails] = pydantic.Field( default_factory=dict ) - staged: dict[StageNr, StagedVersionDetails] = pydantic.Field(default_factory=dict) def extend(self, other: Versions) -> None: assert set(self.model_fields) == {"published", "staged"}, set(self.model_fields) diff --git a/backoffice/validate_format.py b/backoffice/validate_format.py index 393127ac..78fc2ac8 100644 --- a/backoffice/validate_format.py +++ b/backoffice/validate_format.py @@ -260,7 +260,7 @@ def validate_format(staged: StagedVersion): dict( has_dynamic_test_cases=bool(dynamic_test_cases), dynamic_test_cases={"include": dynamic_test_cases}, - version=staged.nr, + version=staged.number, conda_envs=conda_envs, ) ) diff --git a/setup.py b/setup.py index ffd90b20..288e0f77 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ packages=find_packages(exclude=["tests"]), install_requires=[ "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@569666b426cb089503f2ee3bb5651e124d8740e8", # TODO: change to released version - "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@06e6b0f77c696e7c5192fa1340482f97b2df98fc", # TODO: change to released version + "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@f72288dda8cc2c2c5ff3715bcaeb138f4ffe8698", # TODO: change to released version "fire", "loguru", "minio==7.2.4", diff --git a/tests/test_backoffice.py b/tests/test_backoffice.py index 49fc809f..da4fe862 100644 --- a/tests/test_backoffice.py +++ b/tests/test_backoffice.py @@ -1,14 +1,14 @@ import os from backoffice import BackOffice -from backoffice.s3_structure.versions import StageNr +from backoffice.s3_structure.versions import StageNumber def test_backoffice( backoffice: BackOffice, package_url: str, package_id: str, s3_test_folder_url: str ): backoffice.stage(resource_id=package_id, package_url=package_url) - backoffice.test(resource_id=package_id, stage_nr=StageNr(1)) - backoffice.await_review(resource_id=package_id, stage_nr=StageNr(1)) - backoffice.publish(resource_id=package_id, stage_nr=StageNr(1)) + backoffice.test(resource_id=package_id, stage_number=StageNumber(1)) + backoffice.await_review(resource_id=package_id, stage_number=StageNumber(1)) + backoffice.publish(resource_id=package_id, stage_number=StageNumber(1)) backoffice.backup(os.environ["ZENODO_TEST_URL"]) From 55756cd30fbef1c09802e7693554d2b81d129033 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 7 Mar 2024 17:09:47 +0100 Subject: [PATCH 08/13] bump spec and core --- scripts/upload_resource.py | 13 ++++++++----- setup.py | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/upload_resource.py b/scripts/upload_resource.py index f63ecc3d..8de97554 100644 --- a/scripts/upload_resource.py +++ b/scripts/upload_resource.py @@ -16,8 +16,11 @@ load_description, save_bioimageio_package, ) -from bioimageio.spec._internal.utils import get_parent_url -from bioimageio.spec.common import BioimageioYamlContent, BioimageioYamlSource, HttpUrl +from bioimageio.spec.common import ( + BioimageioYamlContent, + BioimageioYamlSource, + RootHttpUrl, +) from dotenv import load_dotenv from minio import Minio from ruyaml import YAML @@ -82,7 +85,7 @@ def upload_resource( def upload_resources( - sources: Sequence[Tuple[BioimageioYamlSource, Union[HttpUrl, Path]]], + sources: Sequence[Tuple[BioimageioYamlSource, Union[RootHttpUrl, Path]]], ): client = Client() # with TemporaryDirectory() as tmp_dir: @@ -134,7 +137,7 @@ def get_model_urls_from_collection_json(): def get_model_urls_from_collection_folder(start: int = 0, end: int = 9999): assert COLLECTION_FOLDER.exists() - ret: List[Tuple[BioimageioYamlContent, HttpUrl]] = [] + ret: List[Tuple[BioimageioYamlContent, RootHttpUrl]] = [] count = 0 for i, resource_path in enumerate( sorted(COLLECTION_FOLDER.glob("**/resource.yaml"))[start:end], start=start @@ -192,7 +195,7 @@ def get_model_urls_from_collection_folder(start: int = 0, end: int = 9999): v += 1 rdf["version"] = v - ret.append((rdf, get_parent_url(rdf_source))) + ret.append((rdf, RootHttpUrl(rdf_source).parent)) count += 1 return ret diff --git a/setup.py b/setup.py index 288e0f77..e2a4aa80 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ ], packages=find_packages(exclude=["tests"]), install_requires=[ - "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@569666b426cb089503f2ee3bb5651e124d8740e8", # TODO: change to released version - "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@f72288dda8cc2c2c5ff3715bcaeb138f4ffe8698", # TODO: change to released version + "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@92d4373e38df0bca93c1463075f15ba0663590e6", # TODO: change to released version + "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@47923d4e02401f886c6216e0d5933b98da81a64f", # TODO: change to released version "fire", "loguru", "minio==7.2.4", From b4d43383e74c575bf11314245cda53e4be3aab21 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 12 Mar 2024 14:41:20 +0100 Subject: [PATCH 09/13] make versions.json version agnostic --- backoffice/_backoffice.py | 4 +-- backoffice/remote_resource.py | 54 ++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/backoffice/_backoffice.py b/backoffice/_backoffice.py index 5e5e86b1..6a32bd5a 100644 --- a/backoffice/_backoffice.py +++ b/backoffice/_backoffice.py @@ -60,9 +60,7 @@ def await_review(self, resource_id: str, stage_number: StageNumber): staged = StagedVersion(self.client, resource_id, stage_number) staged.await_review() - def request_changes( - self, resource_id: str, stage_numbermber: StageNumber, reason: str - ): + def request_changes(self, resource_id: str, stage_number: StageNumber, reason: str): staged = StagedVersion(self.client, resource_id, stage_number) staged.request_changes(reason=reason) diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py index 20c14b60..4c030c66 100644 --- a/backoffice/remote_resource.py +++ b/backoffice/remote_resource.py @@ -35,6 +35,7 @@ yaml = YAML(typ="safe") +VersionSpecificJsonFileT = TypeVar("VersionSpecificJsonFileT", Logs, Chat) JsonFileT = TypeVar("JsonFileT", Versions, Logs, Chat) NumberT = TypeVar("NumberT", StageNumber, PublishNumber) @@ -51,12 +52,17 @@ class RemoteResource: """resource identifier""" @property - def folder(self) -> str: + def resource_folder(self) -> str: """The S3 (sub)prefix of this resource""" - return self.id + return f"{self.id}/" + + @property + def folder(self) -> str: + """The S3 (sub)prefix of this resource (or resource version)""" + return self.resource_folder def get_versions(self) -> Versions: - return self._get_json(Versions) + return self._get_version_agnostic_json(Versions) def get_latest_stage_number(self) -> Optional[StageNumber]: versions = self.get_versions() @@ -84,21 +90,41 @@ def stage_new_version(self, package_url: str) -> StagedVersion: ret.unpack(package_url=package_url) return ret - def _get_json(self, typ: Type[JsonFileT]) -> JsonFileT: - path = f"{self.folder}{typ.__name__.lower()}.json" + def _get_version_agnostic_json(self, typ: Type[Versions]) -> Versions: + return self._get_json(typ, f"{self.resource_folder}{typ.__name__.lower()}.json") + + def _get_version_specific_json( + self, typ: Type[VersionSpecificJsonFileT] + ) -> VersionSpecificJsonFileT: + return self._get_json(typ, f"{self.folder}{typ.__name__.lower()}.json") + + def _get_json(self, typ: Type[JsonFileT], path: str) -> JsonFileT: data = self.client.load_file(path) if data is None: return typ() else: return typ.model_validate_json(data) - def _extend_json( + def _extend_version_agnostic_json( self, - extension: JsonFileT, + extension: Versions, ): - path = f"{self.folder}{extension.__class__.__name__.lower()}.json" + self._extend_json( + extension, + f"{self.resource_folder}{extension.__class__.__name__.lower()}.json", + ) + + def _extend_version_specific_json( + self, + extension: VersionSpecificJsonFileT, + ): + self._extend_json( + extension, f"{self.folder}{extension.__class__.__name__.lower()}.json" + ) + + def _extend_json(self, extension: JsonFileT, path: str): logger.info("Extending {} with {}", path, extension) - current = self._get_json(extension.__class__) + current = self._get_json(extension.__class__, path) _ = current.extend(extension) self.client.put_pydantic(path, current) @@ -128,17 +154,17 @@ def rdf_url(self) -> str: return self.client.get_file_url(f"{self.folder}files/rdf.yaml") def get_log(self) -> Logs: - return self._get_json(Logs) + return self._get_version_specific_json(Logs) def get_chat(self) -> Chat: - return self._get_json(Chat) + return self._get_version_specific_json(Chat) def extend_log( self, extension: Logs, ): """extend log file""" - self._extend_json(extension) + self._extend_version_specific_json(extension) @dataclass @@ -274,7 +300,7 @@ def publish(self) -> PublishedVersion: versions.published[next_publish_nr] = PublishedVersionDetails( sem_ver=sem_ver, status=PublishedStatus(stage_number=self.number) ) - self._extend_json(versions) + self._extend_version_agnostic_json(versions) # TODO: clean up staged files? # remove all uploaded files from this staged version @@ -296,7 +322,7 @@ def _set_status(self, value: StagedVersionStatus): logger.warning("Proceeding from {} to {}", details.status, value) details.status = value - self._extend_json(versions) + self._extend_version_agnostic_json(versions) @dataclass From 1437cfbaa7ac112a93d270e249ca121ff67694e3 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 12 Mar 2024 14:43:08 +0100 Subject: [PATCH 10/13] bump spec and core --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e2a4aa80..949800cf 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ ], packages=find_packages(exclude=["tests"]), install_requires=[ - "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@92d4373e38df0bca93c1463075f15ba0663590e6", # TODO: change to released version - "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@47923d4e02401f886c6216e0d5933b98da81a64f", # TODO: change to released version + "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@b29789236dc9aab7c8a874398adc7fb73b4c9963", # TODO: change to released version + "bioimageio.spec==0.5.0", "fire", "loguru", "minio==7.2.4", From cf1e4208c81651e8b0a055e0444c71a169b663ac Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 12 Mar 2024 14:51:58 +0100 Subject: [PATCH 11/13] ensure we have chat.json and logs.json --- backoffice/remote_resource.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py index 4c030c66..8217ba95 100644 --- a/backoffice/remote_resource.py +++ b/backoffice/remote_resource.py @@ -181,6 +181,13 @@ def version_prefix(self): return "staged/" def unpack(self, package_url: str): + # ensure we have a chat.json + self._extend_version_specific_json(self._get_version_specific_json(Chat)) + + # ensure we have a logs.json + self._extend_version_specific_json(self._get_version_specific_json(Logs)) + + # set first status (this also write versions.json) self._set_status( UnpackingStatus(description=f"unzipping {package_url} to {self.folder}") ) From d732032ae0f646212584dde0c84fbec0a9afd196 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 12 Mar 2024 16:11:18 +0100 Subject: [PATCH 12/13] bump spec and core --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 949800cf..a5a67cf4 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ ], packages=find_packages(exclude=["tests"]), install_requires=[ - "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@b29789236dc9aab7c8a874398adc7fb73b4c9963", # TODO: change to released version - "bioimageio.spec==0.5.0", + "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@63eaae048b2f153926f9250dac85b8ad646edb23", # TODO: change to released version + "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@22a0db9f5b1e2c39f91b88368a312ba5e7810b72", # TODO: change to released version "fire", "loguru", "minio==7.2.4", From bca4a584acb8c514e766b72e37a5da115af41b0a Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 12 Mar 2024 16:24:10 +0100 Subject: [PATCH 13/13] fix pyright issues --- backoffice/validate_format.py | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backoffice/validate_format.py b/backoffice/validate_format.py index 78fc2ac8..6df66ace 100644 --- a/backoffice/validate_format.py +++ b/backoffice/validate_format.py @@ -4,10 +4,10 @@ import pooch from bioimageio.spec import InvalidDescr, ResourceDescr, load_description +from bioimageio.spec.common import Sha256 from bioimageio.spec.model import v0_4, v0_5 -from bioimageio.spec.model.v0_5 import WeightsFormat +from bioimageio.spec.model.v0_5 import Version, WeightsFormat from bioimageio.spec.summary import ErrorEntry, ValidationDetail -from packaging.version import Version from ruyaml import YAML from typing_extensions import assert_never @@ -51,7 +51,7 @@ def get_env_from_deps(deps: Union[v0_4.Dependencies, v0_5.EnvironmentFileDescr]) return get_base_env() url = deps.file - sha = None + sha: Optional[Sha256] = None elif isinstance(deps, v0_5.EnvironmentFileDescr): url = deps.source sha = deps.sha256 diff --git a/setup.py b/setup.py index a5a67cf4..1be89427 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ ], packages=find_packages(exclude=["tests"]), install_requires=[ - "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@63eaae048b2f153926f9250dac85b8ad646edb23", # TODO: change to released version + "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@185b7e8199852391cd80384829f4f515df3ec13e", # TODO: change to released version "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@22a0db9f5b1e2c39f91b88368a312ba5e7810b72", # TODO: change to released version "fire", "loguru",