feat(import): add import from a path (#6)

docker/backend.dockerfile

@@ -6,9 +6,9 @@ ARG OPENJPEG_VERSION=2.4.0
 ARG PIMS_REVISION
 ARG PIMS_VERSION
 ARG PLUGIN_CSV=scripts/plugin-list.csv
-ARG PY_VERSION=3.8
+ARG PY_VERSION=3.10
 ARG SETUPTOOLS_VERSION=59.6.0
-ARG UBUNTU_VERSION=20.04
+ARG UBUNTU_VERSION=22.04
 ARG VIPS_URL=https://github.com/libvips/libvips/releases/download
 ARG VIPS_VERSION=8.12.1
 
@@ -23,7 +23,7 @@ FROM ubuntu:${UBUNTU_VERSION}
 ENV LANG C.UTF-8
 ENV DEBIAN_FRONTEND noninteractive
 
-ARG PY_VERSION=3.8
+ARG PY_VERSION=3.10
 
 RUN apt-get -y update && apt-get -y install --no-install-recommends --no-install-suggests \
         `# Essentials` \
@@ -167,7 +167,7 @@ ARG PIMS_PACKAGE_REVISION
 ARG PIMS_PACKAGE_VERSION
 ARG PIMS_VERSION
 ARG PLUGIN_CSV=scripts/plugin-list.csv
-ARG PY_VERSION=3.8
+ARG PY_VERSION=3.10
 ARG SETUPTOOLS_VERSION=59.6.0
 ARG UBUNTU_VERSION=20.04
 ARG VIPS_URL=https://github.com/libvips/libvips/releases/download

pims-config.env

@@ -4,6 +4,8 @@ WRITING_PATH="/data/pims/tmp"
 DEFAULT_IMAGE_SIZE_SAFETY_MODE="SAFE_REJECT"
 DEFAULT_ANNOTATION_ORIGIN="LEFT_TOP"
 OUTPUT_SIZE_LIMIT="10000"
+CRYPT4GH_PUBLIC_KEY=""
+CRYPT4GH_PRIVATE_KEY=""
 CYTOMINE_PUBLIC_KEY=""
 CYTOMINE_PRIVATE_KEY=""
 PIMS_URL="http://localhost-ims"

pims-dev-config.env

@@ -4,6 +4,8 @@ WRITING_PATH="/data/pims/tmp"
 DEFAULT_IMAGE_SIZE_SAFETY_MODE=SAFE_REJECT
 DEFAULT_ANNOTATION_ORIGIN=LEFT_TOP
 OUTPUT_SIZE_LIMIT=10000
+CRYPT4GH_PUBLIC_KEY="abc"
+CRYPT4GH_PRIVATE_KEY="def"
 CYTOMINE_PUBLIC_KEY=imageServerPublicKey
 CYTOMINE_PRIVATE_KEY=imageServerPrivateKey
 PIMS_URL=http://ims.cytomine.local

pims/api/operations.py

@@ -11,35 +11,47 @@
 #  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  * See the License for the specific language governing permissions and
 #  * limitations under the License.
+
 import logging
 import os
 import traceback
 from typing import Optional
-import aiofiles
 
+import aiofiles
 from cytomine import Cytomine
-from cytomine.models import (
-    Project, ProjectCollection, Storage, UploadedFile
+from cytomine.models import Project, ProjectCollection, Storage, UploadedFile
+from fastapi import APIRouter, BackgroundTasks, Depends, Query
+from starlette.formparsers import (
+    MultiPartMessage,
+    MultiPartParser,
+    _user_safe_decode,
 )
-from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, Query, UploadFile
 from starlette.requests import Request
 from starlette.responses import FileResponse, JSONResponse
-from starlette.formparsers import MultiPartMessage, MultiPartParser, _user_safe_decode
 
 from pims.api.exceptions import (
-    AuthenticationException, BadRequestException, CytomineProblem,
-    check_representation_existence
+    AuthenticationException,
+    BadRequestException,
+    CytomineProblem,
+    NotFoundException,
+    check_representation_existence,
 )
 from pims.api.utils.cytomine_auth import (
-    get_this_image_server, parse_authorization_header,
-    parse_request_token, sign_token
+    get_this_image_server,
+    parse_authorization_header,
+    parse_request_token,
+    sign_token,
+)
+from pims.api.utils.parameter import (
+    filepath_parameter,
+    imagepath_parameter,
+    sanitize_filename,
 )
-from pims.api.utils.parameter import filepath_parameter, imagepath_parameter, sanitize_filename
 from pims.api.utils.response import serialize_cytomine_model
 from pims.config import Settings, get_settings
 from pims.files.archive import make_zip_archive
 from pims.files.file import Path
-from pims.importer.importer import run_import
+from pims.importer.importer import run_import, run_import_from_path
 from pims.importer.listeners import CytomineListener
 from pims.tasks.queue import Task, send_task
 from pims.utils.iterables import ensure_list
@@ -56,8 +68,78 @@
 
 cytomine_logger = logging.getLogger("pims.cytomine")
 
+REQUIRED_DIRECTORIES = ["images", "metadata"]
 WRITING_PATH = get_settings().writing_path
 
+
+def is_dataset_structured(dataset_path: str) -> bool:
+    """Check the structure of a dataset."""
+
+    missing_directories = [
+        directory
+        for directory in REQUIRED_DIRECTORIES
+        if not os.path.isdir(os.path.join(dataset_path, directory))
+    ]
+
+    return missing_directories == []
+
+
+@router.post("/import", tags=["Import"])
+def import_dataset(
+    request: Request,
+    host: str = Query(..., description="The Cytomine host"),
+    path: str = Query(..., description="The absolute path to the datasets to import"),
+    storage_id: int = Query(..., description="The storage where to import the dataset"),
+    config: Settings = Depends(get_settings)
+) -> JSONResponse:
+    """Import a dataset from a given absolute path."""
+
+    if not storage_id:
+        raise BadRequestException(detail="'storage' parameter is missing.")
+
+    if not os.path.exists(path):
+        raise NotFoundException(detail="The provided dataset path does not exist.")
+
+    datasets = [
+        dataset_path
+        for dataset in os.listdir(path)
+        if (dataset_path := os.path.join(path, dataset))
+        and is_dataset_structured(dataset_path)
+    ]
+
+    public_key, signature = parse_authorization_header(request.headers)
+    cytomine_auth = (host, config.cytomine_public_key, config.cytomine_private_key)
+
+    with Cytomine(*cytomine_auth, configure_logging=False) as c:
+        if not c.current_user:
+            raise AuthenticationException("PIMS authentication to Cytomine failed.")
+
+        this = get_this_image_server(config.pims_url)
+        cyto_keys = c.get(f"userkey/{public_key}/keys.json")
+        private_key = cyto_keys["privateKey"]
+
+        if sign_token(private_key, parse_request_token(request)) != signature:
+            raise AuthenticationException("Authentication to Cytomine failed")
+
+        c.set_credentials(public_key, private_key)
+        user = c.current_user
+
+        storage = Storage().fetch(storage_id)
+        if not storage:
+            raise CytomineProblem(f"Storage {storage_id} not found")
+
+    for dataset in datasets:
+        run_import_from_path(
+            dataset,
+            cytomine_auth,
+            storage_id,
+            this.id,
+            user.id,
+        )
+
+    return JSONResponse(content={"status": "ok"})
+
+
 @router.post('/upload', tags=['Import'])
 async def import_direct_chunks(
     request: Request,

pims/config.py

@@ -23,6 +23,7 @@
 
 class ReadableSettings(BaseSettings):
     root: str
+    dataset_path: str = "/dataset"
     pending_path: str = "/tmp/uploaded"
     writing_path: str = "/data/pims/tmp"
     checker_resolution_file: str = "checkerResolution.csv"
@@ -56,6 +57,9 @@ class Config:
 
 
 class Settings(ReadableSettings):
+    crypt4gh_public_key: str
+    crypt4gh_private_key: str
+
     cytomine_public_key: str
     cytomine_private_key: str
 

pims/files/file.py

@@ -42,7 +42,7 @@
 SPECTRAL_STEM = "spectral"
 HISTOGRAM_STEM = "histogram"
 
-_NUM_SIGNATURE_BYTES = 262
+NUM_SIGNATURE_BYTES = 262
 
 
 class FileRole(str, Enum):
@@ -367,7 +367,7 @@ def signature(self) -> bytearray:
         if not self.is_file():
             return bytearray()
         with self.resolve().open('rb') as fp:
-            return bytearray(fp.read(_NUM_SIGNATURE_BYTES))
+            return bytearray(fp.read(NUM_SIGNATURE_BYTES))
 
     @property
     def path(self) -> Path:

pims/formats/common/dicom.py

@@ -11,15 +11,15 @@
 #  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  * See the License for the specific language governing permissions and
 #  * limitations under the License.
+
 import logging
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import List, Optional
 
 import numpy as np
 import pyvips
 from pint import Quantity
 from pydicom import FileDataset, dcmread
-from pydicom.dicomdir import DicomDir
 from pydicom.multival import MultiValue
 from pydicom.uid import ImplicitVRLittleEndian
 from pyvips import GValue
@@ -56,9 +56,12 @@ def _pydicom_dcmread(path, *args, **kwargs):
     return dcm
 
 
-def cached_dcmread(format: AbstractFormat) -> Union[FileDataset, DicomDir]:
+def cached_dcmread(format: AbstractFormat) -> FileDataset:
     return format.get_cached(
-        '_dcmread', _pydicom_dcmread, format.path.resolve(), force=True
+        '_dcmread',
+        _pydicom_dcmread,
+        format.path.resolve(),
+        force=True,
     )
 
 

pims/formats/utils/abstract.py

@@ -19,6 +19,7 @@
 from typing import Any, Dict, List, Optional, TYPE_CHECKING, Type
 
 from pims.cache import SimpleDataCache, cached_property
+from pims.config import get_settings
 from pims.formats.utils.checker import AbstractChecker
 from pims.formats.utils.convertor import AbstractConvertor
 from pims.formats.utils.histogram import AbstractHistogramReader
@@ -89,6 +90,17 @@ def __init__(self, path: Path, existing_cache: Dict[str, Any] = None):
 
         self.histogram_reader = self.histogram_reader_class(self)
 
+        settings = get_settings()
+        credentials = {
+            "public_key": settings.crypt4gh_public_key,
+            "private_key": settings.crypt4gh_private_key,
+        }
+
+        for component in (self.parser, self.reader):
+            if hasattr(component, "set_credentials"):
+                component.set_credentials(credentials)
+
+
     @classmethod
     def init(cls):
         """
@@ -184,6 +196,13 @@ def match(cls, cached_path: CachedDataPath) -> bool:
             Whether it is this format
         """
         if cls.checker_class:
+            if hasattr(cls.checker_class, "CREDENTIALS"):
+                settings = get_settings()
+                cls.checker_class.CREDENTIALS = {
+                    "public_key": settings.crypt4gh_public_key,
+                    "private_key": settings.crypt4gh_private_key,
+                }
+
             return cls.checker_class.match(cached_path)
         return False