diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a909b9a1..7466aa78 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,4 +1,4 @@ -name: test stage call +name: test and build docs on: push diff --git a/README.md b/README.md index 4320c0c6..f0478bb1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # collection + Maintains the resources displayed on bioimage.io (Successor to collection-bioimage-io) diff --git a/backoffice/VERSION b/backoffice/VERSION new file mode 100644 index 00000000..41de6987 --- /dev/null +++ b/backoffice/VERSION @@ -0,0 +1,3 @@ +{ + "version": "0.1.0" +} diff --git a/scripts/utils/__init__.py b/backoffice/__init__.py similarity index 100% rename from scripts/utils/__init__.py rename to backoffice/__init__.py diff --git a/backoffice/__main__.py b/backoffice/__main__.py new file mode 100644 index 00000000..f79717f7 --- /dev/null +++ b/backoffice/__main__.py @@ -0,0 +1,55 @@ +import os +from typing import Literal, Optional + +import fire +from bioimageio.spec.model.v0_5 import WeightsFormat +from dotenv import load_dotenv + +from .backup import backup +from .run_dynamic_tests import run_dynamic_tests +from .utils.remote_resource import PublishedVersion, RemoteResource, StagedVersion +from .utils.s3_client import Client +from .validate_format import validate_format + +_ = load_dotenv() + + +class BackOffice: + def __init__(self) -> None: + super().__init__() + self.client = Client() + + def stage(self, resource_id: str, package_url: str): + resource = RemoteResource(client=Client(), id=resource_id) + staged = resource.stage_new_version(package_url) + validate_format(staged) + + def test( + self, + resource_id: str, + version: int, + weight_format: Optional[WeightsFormat] = None, + create_env_outcome: Literal["success", ""] = "success", + ): + staged = StagedVersion(self.client, resource_id, version) + run_dynamic_tests( + staged=staged, + weight_format=weight_format, + create_env_outcome=create_env_outcome, + ) + + def await_review(self, resource_id: str, version: int): + staged = StagedVersion(self.client, resource_id, version) + staged.await_review() + + def publish(self, resource_id: str, stage_nr: int): + staged = StagedVersion(client=self.client, id=resource_id, version=stage_nr) + published = staged.publish() + assert isinstance(published, PublishedVersion) + + def backup(self): + _ = backup(self.client, os.environ["ZENODO_URL"]) + + +if __name__ == "__main__": + fire.Fire(BackOffice) diff --git a/scripts/backup.py b/backoffice/backup.py similarity index 51% rename from scripts/backup.py rename to backoffice/backup.py index 9b5cb991..0fb4047f 100644 --- a/scripts/backup.py +++ b/backoffice/backup.py @@ -1,25 +1,19 @@ import os -import typer from dotenv import load_dotenv from loguru import logger -from utils.s3_client import Client + +from backoffice.utils.s3_client import Client _ = load_dotenv() -def backup(): +def backup(client: Client, destination: str): """backup collection Returns: list of folders and file names backed up """ - client = Client() content_to_backup = list(client.ls("")) - destination = os.environ["ZENODO_URL"] - logger.error("Backup to '{}': {}", destination, content_to_backup) + logger.error("Not implemented: Backup to '{}': {}", destination, content_to_backup) return content_to_backup - - -if __name__ == "__main__": - typer.run(backup) diff --git a/scripts/run_dynamic_tests.py b/backoffice/run_dynamic_tests.py similarity index 88% rename from scripts/run_dynamic_tests.py rename to backoffice/run_dynamic_tests.py index f3fea8b7..bc6fd65f 100644 --- a/scripts/run_dynamic_tests.py +++ b/backoffice/run_dynamic_tests.py @@ -5,7 +5,6 @@ import bioimageio.core import bioimageio.spec -import typer from bioimageio.spec.model.v0_5 import WeightsFormat from bioimageio.spec.summary import ( ErrorEntry, @@ -14,8 +13,8 @@ ValidationSummary, ) from ruyaml import YAML -from utils.remote_resource import StagedVersion -from utils.s3_client import Client + +from backoffice.utils.remote_resource import StagedVersion try: from tqdm import tqdm @@ -44,14 +43,10 @@ def get_summary_detail_from_exception(name: str, exception: Exception): def run_dynamic_tests( - resource_id: str, - version: int, - weight_format: Optional[WeightsFormat] = typer.Argument( - ..., help="weight format to test model with." - ), - create_env_outcome: str = "success", + staged: StagedVersion, + weight_format: Optional[WeightsFormat], # "weight format to test model with." + create_env_outcome: str, ): - staged = StagedVersion(client=Client(), id=resource_id, version=version) staged.set_status( "testing", "Testing" + ("" if weight_format is None else f" {weight_format} weights"), @@ -124,7 +119,3 @@ def run_dynamic_tests( ) staged.add_log_entry("bioimageio.core", summary.model_dump(mode="json")) - - -if __name__ == "__main__": - typer.run(run_dynamic_tests) diff --git a/scripts/upload_model_to_zenodo.py b/backoffice/upload_model_to_zenodo.py similarity index 99% rename from scripts/upload_model_to_zenodo.py rename to backoffice/upload_model_to_zenodo.py index 0f715b2d..861aa5ec 100644 --- a/scripts/upload_model_to_zenodo.py +++ b/backoffice/upload_model_to_zenodo.py @@ -15,7 +15,7 @@ from packaging.version import parse as parse_version from ruyaml import YAML -# from utils.s3_client import create_client, version_from_resource_path_or_s3 +# from backoffice.utils.s3_client import create_client, version_from_resource_path_or_s3 yaml = YAML(typ="safe") diff --git a/backoffice/utils/__init__.py b/backoffice/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/utils/remote_resource.py b/backoffice/utils/remote_resource.py similarity index 97% rename from scripts/utils/remote_resource.py rename to backoffice/utils/remote_resource.py index c742408b..108893be 100644 --- a/scripts/utils/remote_resource.py +++ b/backoffice/utils/remote_resource.py @@ -184,6 +184,12 @@ def _set_log(self, log: Log) -> None: class StagedVersion(_RemoteResourceVersion): version_prefix: ClassVar[str] = "staged/" + def await_review(self): + self.set_status( + "awaiting review", + description="Thank you for your contribution! Our bioimage.io maintainers will take a look soon.", + ) + def publish(self) -> PublishedVersion: # get next version and update versions.json versions_path = f"{self.id}/versions.json" diff --git a/scripts/utils/s3_client.py b/backoffice/utils/s3_client.py similarity index 100% rename from scripts/utils/s3_client.py rename to backoffice/utils/s3_client.py diff --git a/scripts/utils/s3_structure.py b/backoffice/utils/s3_structure.py similarity index 100% rename from scripts/utils/s3_structure.py rename to backoffice/utils/s3_structure.py diff --git a/scripts/utils/validate_format.py b/backoffice/validate_format.py similarity index 99% rename from scripts/utils/validate_format.py rename to backoffice/validate_format.py index a750e95e..f29ec1bf 100644 --- a/scripts/utils/validate_format.py +++ b/backoffice/validate_format.py @@ -11,7 +11,7 @@ from packaging.version import Version from ruyaml import YAML -from .remote_resource import StagedVersion +from backoffice.utils.remote_resource import StagedVersion yaml = YAML(typ="safe") diff --git a/py.typed b/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/pyproject.toml b/pyproject.toml index 5c7372e3..157be5d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ target-version = ["py312"] [tool.pyright] -include = ["scripts"] +include = ["backoffice", "tests"] pythonPlatform = "All" pythonVersion = "3.12" reportIncompatibleMethodOverride = true @@ -20,4 +20,4 @@ useLibraryCodeForTypes = true [tool.pytest.ini_options] addopts = "--capture=no --doctest-modules --failed-first" -testpaths = ["scripts", "tests"] +testpaths = ["backoffice", "tests"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 897afc6f..00000000 --- a/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@19105665ad779014e03c7b311c0b4003ab08f752 # TODO: chenage to released version -bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@3a7875b5debc2d52b2fc87f6579afe217e1c7280 # TODO: chenage to released version -black==24.2.0 -loguru==0.7.2 -minio==7.2.3 -packaging==23.2 -pdoc==14.4.0 -pyright==1.1.351 -pytest==8.0.0 -ruyaml==0.91.0 -spdx-license-list==3.22 -typer==0.9.0 diff --git a/scripts/conclude.py b/scripts/conclude.py deleted file mode 100644 index b8f85558..00000000 --- a/scripts/conclude.py +++ /dev/null @@ -1,18 +0,0 @@ -from typer import run -from utils.remote_resource import StagedVersion -from utils.s3_client import Client - - -def conclude( - resource_id: str, - version: int, -): - staged = StagedVersion(client=Client(), id=resource_id, version=version) - staged.set_status( - "awaiting review", - description="Thank you for your contribution! Our bioimage.io maintainers will take a look soon.", - ) - - -if __name__ == "__main__": - run(conclude) diff --git a/scripts/publish.py b/scripts/publish.py deleted file mode 100644 index 4e70106c..00000000 --- a/scripts/publish.py +++ /dev/null @@ -1,13 +0,0 @@ -import typer -from utils.remote_resource import PublishedVersion, StagedVersion -from utils.s3_client import Client - - -def publish(resource_id: str, stage_nr: int): - staged = StagedVersion(client=Client(), id=resource_id, version=stage_nr) - published = staged.publish() - assert isinstance(published, PublishedVersion) - - -if __name__ == "__main__": - typer.run(publish) diff --git a/scripts/stage.py b/scripts/stage.py deleted file mode 100644 index 1e1f8fbc..00000000 --- a/scripts/stage.py +++ /dev/null @@ -1,14 +0,0 @@ -import typer -from utils.remote_resource import RemoteResource -from utils.s3_client import Client -from utils.validate_format import validate_format - - -def stage(resource_id: str, package_url: str): - resource = RemoteResource(client=Client(), id=resource_id) - staged = resource.stage_new_version(package_url) - validate_format(staged) - - -if __name__ == "__main__": - typer.run(stage) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..2e163fba --- /dev/null +++ b/setup.py @@ -0,0 +1,50 @@ +import json +from pathlib import Path + +from setuptools import find_packages, setup + +# Get the long description from the README file +ROOT_DIR = Path(__file__).parent.resolve() +long_description = (ROOT_DIR / "README.md").read_text(encoding="utf-8") +VERSION_FILE = ROOT_DIR / "backoffice" / "VERSION" +VERSION = json.loads(VERSION_FILE.read_text(encoding="utf-8"))["version"] + + +setup( + name="backoffice", + version=VERSION, + description="backoffice to control bioimage.io collection", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/bioimage-io/collection", + author="bioimage.io Team", + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.12", + ], + packages=find_packages(exclude=["tests"]), + install_requires=[ + "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@3a7875b5debc2d52b2fc87f6579afe217e1c7280", # TODO: change to released version + "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@19105665ad779014e03c7b311c0b4003ab08f752", # TODO: change to released version + "fire", + "loguru", + "minio==7.2.3", + "ruyaml", + "tqdm", + ], + extras_require={ + "dev": [ + "black", + "pdoc", + "pre-commit", + "pyright", + "pytest", + ] + }, + entry_points={"console_scripts": ["backoffice = backoffice.__main__"]}, + project_urls={ + "Bug Reports": "https://github.com/bioimage-io/collection/issues", + "Source": "https://github.com/bioimage-io/collection", + }, +) diff --git a/tests/test_scripts/test_utils/test_remote_resource.py b/tests/test_utils/test_remote_resource.py similarity index 63% rename from tests/test_scripts/test_utils/test_remote_resource.py rename to tests/test_utils/test_remote_resource.py index 76b03194..8c5c56df 100644 --- a/tests/test_scripts/test_utils/test_remote_resource.py +++ b/tests/test_utils/test_remote_resource.py @@ -1,19 +1,17 @@ -from typing import TYPE_CHECKING +import os -if TYPE_CHECKING: - from scripts.utils.s3_client import Client +from backoffice.backup import backup +from backoffice.utils.remote_resource import ( + PublishedVersion, + RemoteResource, + StagedVersion, +) +from backoffice.utils.s3_client import Client def test_lifecycle( - client: "Client", package_url: str, package_id: str, s3_test_folder_url: str + client: Client, package_url: str, package_id: str, s3_test_folder_url: str ): - from scripts.backup import backup - from scripts.utils.remote_resource import ( - PublishedVersion, - RemoteResource, - StagedVersion, - ) - resource = RemoteResource(client=client, id=package_id) staged = resource.stage_new_version(package_url) assert isinstance(staged, StagedVersion) @@ -29,5 +27,5 @@ def test_lifecycle( published_rdf_url == f"{s3_test_folder_url}frank-water-buffalo/1/files/rdf.yaml" ) - backed_up = backup() + backed_up = backup(client, os.environ["ZENODO_TEST"]) assert backed_up == ["frank-water-buffalo"] diff --git a/tests/test_scripts/test_utils/test_s3client.py b/tests/test_utils/test_s3client.py similarity index 81% rename from tests/test_scripts/test_utils/test_s3client.py rename to tests/test_utils/test_s3client.py index 2e5117d8..9776212e 100644 --- a/tests/test_scripts/test_utils/test_s3client.py +++ b/tests/test_utils/test_s3client.py @@ -1,10 +1,7 @@ -from typing import TYPE_CHECKING +from backoffice.utils.s3_client import Client -if TYPE_CHECKING: - from scripts.utils.s3_client import Client - -def test_client(client: "Client"): +def test_client(client: Client): assert client.prefix.startswith("sandbox") client.put_json("test/test1.json", "test") client.put_json("test/dir/test2.json", "test")