Feature/zenodo uploader

.gitignore

@@ -2,6 +2,9 @@
 # data/ is intended for database data from the mysql container
 data/
 
+# MacOS 
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

src/main.py

@@ -13,15 +13,14 @@
 from sqlalchemy.engine import Engine
 from sqlmodel import Session, select
 
-import routers
 from authentication import get_current_user
 from config import KEYCLOAK_CONFIG
 from database.deletion.triggers import add_delete_triggers
 from database.model.concept.concept import AIoDConcept
 from database.model.platform.platform import Platform
 from database.model.platform.platform_names import PlatformName
 from database.setup import sqlmodel_engine
-from routers import resource_routers, parent_routers, enum_routers
+from routers import resource_routers, parent_routers, enum_routers, uploader_routers
 
 
 def _parse_args() -> argparse.Namespace:
@@ -78,9 +77,9 @@ def counts() -> dict:
 
     for router in (
         resource_routers.router_list
-        + routers.other_routers
         + parent_routers.router_list
         + enum_routers.router_list
+        + uploader_routers.router_list
     ):
         app.include_router(router.create(engine, url_prefix))
 

src/routers/__init__.py

@@ -1,5 +1 @@
 from .resource_router import ResourceRouter  # noqa:F401
-from .upload_router_huggingface import UploadRouterHuggingface
-
-
-other_routers = [UploadRouterHuggingface()]

src/routers/uploader_router.py

@@ -0,0 +1,11 @@
+import abc
+
+from sqlalchemy.engine import Engine
+from fastapi import APIRouter
+
+
+class UploaderRouter(abc.ABC):
+    def create(self, engine: Engine, url_prefix: str) -> APIRouter:
+        router = APIRouter()
+
+        return router

src/routers/uploader_routers/__init__.py

@@ -0,0 +1,5 @@
+from routers.uploader_router import UploaderRouter
+from .upload_router_zenodo import UploadRouterZenodo
+from .upload_router_huggingface import UploadRouterHuggingface
+
+router_list: list[UploaderRouter] = [UploadRouterZenodo(), UploadRouterHuggingface()]

src/routers/upload_router_huggingface.py → src/routers/uploader_routers/upload_router_huggingface.py

@@ -2,12 +2,13 @@
 from fastapi import File, Query, UploadFile
 from sqlalchemy.engine import Engine
 
-from uploader.hugging_face_uploader import handle_upload
+from uploaders.hugging_face_uploader import handle_upload
+from routers.uploader_router import UploaderRouter
 
 
-class UploadRouterHuggingface:
+class UploadRouterHuggingface(UploaderRouter):
     def create(self, engine: Engine, url_prefix: str) -> APIRouter:
-        router = APIRouter()
+        router = super().create(engine, url_prefix)
 
         @router.post(url_prefix + "/upload/datasets/{identifier}/huggingface", tags=["upload"])
         def huggingFaceUpload(

src/routers/uploader_routers/upload_router_zenodo.py

@@ -0,0 +1,37 @@
+from typing import Annotated
+
+from fastapi import APIRouter
+from fastapi import File, Query, UploadFile, Path
+from sqlalchemy.engine import Engine
+
+from uploaders.zenodo_uploader import ZenodoUploader
+from routers.uploader_router import UploaderRouter
+
+
+class UploadRouterZenodo(UploaderRouter):
+    def create(self, engine: Engine, url_prefix: str) -> APIRouter:
+        router = super().create(engine, url_prefix)
+
+        zenodo_uploader = ZenodoUploader()
+
+        @router.post(url_prefix + "/upload/datasets/{identifier}/zenodo", tags=["upload"])
+        def zenodo_upload(
+            identifier: int = Path(
+                description="The AIoD dataset identifier",
+            ),
+            file: UploadFile = File(
+                title="File", description="This file will be uploaded to Zenodo"
+            ),
+            publish: Annotated[
+                bool,
+                Query(
+                    title="Publish dataset",
+                    description="When published, the dataset and files will be publicaly ccessible "
+                    "and you will no longer be able to upload more files!",
+                ),
+            ] = False,
+            token: str = Query(title="Zenodo Token", description="The access token of Zenodo"),
+        ) -> int:
+            return zenodo_uploader.handle_upload(engine, identifier, publish, token, file)
+
+        return router

src/tests/uploader/zenodo/mock_zenodo.py

@@ -0,0 +1,139 @@
+"""
+    Generates mocked responses for zenodo endpoints.
+    - POST to BASE_URL: Creates an empty record
+    - GET to BASE_URL/RESOURCE_ID: Gets the info of the record.
+    - PUT to BASE_URL/REOURCE_ID: Updates metadata
+    - PUT to REPO_URL/FILE_NAME: Uploads a file
+    - POST to BASE_URL/REOURCE_ID/actions/publish: Publishes the dataset with all content
+    - GET to BASE_URL/RESOURCE_ID/files: Gets the list of files in draft mode
+    - GET to RECORDS_URL/RESOURCE_ID/files: Gets the list of published data
+"""
+
+import responses
+
+BASE_URL = "https://zenodo.org/api/deposit/depositions"
+REPO_URL = "https://zenodo.org/api/files/fake-bucket-id00"
+RECORDS_URL = "https://zenodo.org/api/records"
+RESOURCE_ID = 100
+
+
+def mock_create_repo(mocked_requests: responses.RequestsMock) -> responses.RequestsMock:
+    mocked_requests.add(
+        responses.POST,
+        BASE_URL,
+        json=record_response(),
+        status=201,
+    )
+
+    return mocked_requests
+
+
+def mock_get_repo_metadata(
+    mocked_requests: responses.RequestsMock, is_published: bool = False
+) -> responses.RequestsMock:
+    mocked_requests.add(
+        responses.GET,
+        f"{BASE_URL}/{RESOURCE_ID}",
+        json=record_response(is_published),
+        status=200,
+    )
+
+    return mocked_requests
+
+
+def mock_update_metadata(mocked_requests: responses.RequestsMock) -> responses.RequestsMock:
+    mocked_requests.add(
+        responses.PUT,
+        f"{BASE_URL}/{RESOURCE_ID}",
+        json={},
+        status=200,
+    )
+    return mocked_requests
+
+
+def mock_upload_file(
+    mocked_requests: responses.RequestsMock, new_file: str
+) -> responses.RequestsMock:
+    mocked_requests.add(
+        responses.PUT,
+        f"{REPO_URL}/{new_file}",
+        json={},
+        status=201,
+    )
+    return mocked_requests
+
+
+def mock_publish_resource(mocked_requests: responses.RequestsMock) -> responses.RequestsMock:
+    mocked_requests.add(
+        responses.POST,
+        f"{BASE_URL}/{RESOURCE_ID}/actions/publish",
+        json=publish_response(),
+        status=202,
+    )
+    return mocked_requests
+
+
+def mock_get_draft_files(
+    mocked_requests: responses.RequestsMock, files: list[str]
+) -> responses.RequestsMock:
+    mocked_requests.add(
+        responses.GET,
+        f"{BASE_URL}/{RESOURCE_ID}/files",
+        json=draft_files_response(files),
+        status=200,
+    )
+    return mocked_requests
+
+
+def mock_get_published_files(
+    mocked_requests: responses.RequestsMock, files: list[str]
+) -> responses.RequestsMock:
+    mocked_requests.add(
+        responses.GET,
+        f"{RECORDS_URL}/{RESOURCE_ID}/files",
+        json=published_files_reponse(files),
+        status=200,
+    )
+    return mocked_requests
+
+
+def record_response(is_published: bool = False) -> dict:
+    response = {
+        "id": RESOURCE_ID,
+        # just the state `done` matters here
+        "state": "done" if is_published else "unsubmitted/inprogress",
+        "links": {"bucket": REPO_URL, "record": f"{RECORDS_URL}/{RESOURCE_ID}"},
+    }
+    return response
+
+
+def publish_response() -> dict:
+    response = {"links": {"record": f"{RECORDS_URL}/{RESOURCE_ID}"}}
+    return response
+
+
+def draft_files_response(filenames: list[str]) -> list[dict]:
+    """Truncated reponse from zenodo when a request is made to the draft repo url."""
+    response = [
+        {"id": f"123-{name}", "filename": name, "filesize": 20, "checksum": "12345abcd"}
+        for name in filenames
+    ]
+    return response
+
+
+def published_files_reponse(filenames: list[str]) -> dict[str, list[dict]]:
+    """Truncated reponse from zenodo when a request is made to the public repo url."""
+    response = {
+        "entries": [
+            {
+                "key": name,
+                "file_id": f"123-{name}",
+                "checksum": "12345abcd",
+                "size": 20,
+                "links": {"content": f"{RECORDS_URL}/{RESOURCE_ID}/files/{name}/content"},
+            }
+            for name in filenames
+        ]
+    }
+
+    return response