Merge pull request #1 from Australian-Imaging-Service/tom-dev

Implements the upload command line

.github/workflows/tests.yml

@@ -48,6 +48,10 @@ jobs:
         echo "machine localhost:8080" > $HOME/.netrc
         echo "user admin" >> $HOME/.netrc
         echo "password admin" >> $HOME/.netrc
-
     - name: Pytest
-      run: pytest -vvs
+      run: pytest -vvs --cov xnat_ingest --cov-config .coveragerc --cov-report xml .
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v2
+      with:
+        fail_ci_if_error: true
+        token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -21,4 +21,4 @@ __pycache__
 *.egg-info
 ~*
 *.venv
-/xnat_siemens_export_upload/_version.py
+/xnat_ingest/_version.py

Dockerfile

@@ -5,6 +5,7 @@ RUN apt-get update && apt-get install -y \
   python3-pip \
   git \
   dcmtk \
+  mrtrix3 \
   && rm -rf /var/lib/apt/lists/*
 
 # Add application code
@@ -14,4 +15,4 @@ RUN pip3 install /app
 
 
 # Set application entrypoint to docker entrypoint
-ENTRYPOINT ["xnat-siemens-export-upload"]
+ENTRYPOINT ["xnat-ingest"]

README.md

@@ -1,18 +1,19 @@
 
-# xnat-siemens-export-upload
+# Xnat-ingest
 
-[![tests](https://github.com/Australian-Imaging-Service/xnat-siemens-export-upload/actions/workflows/tests.yml/badge.svg)](https://github.com/Australian-Imaging-Service/xnat-siemens-export-upload/actions/workflows/tests.yml)
+[![tests](https://github.com/Australian-Imaging-Service/xnat-ingest/actions/workflows/tests.yml/badge.svg)](https://github.com/Australian-Imaging-Service/xnat-ingest/actions/workflows/tests.yml)
+[![coverage](https://codecov.io/gh/arcanaframework/xnat-ingest/branch/main/graph/badge.svg?token=UIS0OGPST7)](https://codecov.io/gh/Australian-Imaging-Service/xnat-ingest)
 
-Upload exported DICOM and list-mode data from Siemens Quadra "Total Body" PET scanner to
-XNAT
+De-identify and upload exported DICOM and associated data files to XNAT based on ID values
+stored within the DICOM headers.
 
 
 ## Installation
 
 Build the docker image from the root directory of a clone of this code repository
 
 ```
-docker build -t xnat-siemens-export-upload .
+docker build -t xnat-ingest .
 ```
 
 
@@ -22,6 +23,6 @@ The root CLI command is set to be the entrypoint of the Docker image so it can b
 by
 
 ```
-docker run xnat-siemens-export-upload --help
+docker run xnat-ingest --help
 ```
 ```

conftest.py

@@ -1,16 +1,13 @@
 import os
-import logging
 from pathlib import Path
-import shutil
+import logging
+import tempfile
 from logging.handlers import SMTPHandler
 import pytest
 from click.testing import CliRunner
 import xnat4tests
-from medimages4tests.dummy.dicom.pet.tbp.siemens.quadra.s7vb10b import (
-    get_image,
-    get_raw_data_files,
-)
-from xnat_exported_scans.utils import logger
+from datetime import datetime
+from xnat_ingest.utils import logger
 
 # Set DEBUG logging for unittests
 
@@ -44,6 +41,12 @@ def catch_cli_exceptions():
     return CATCH_CLI_EXCEPTIONS
 
 
+@pytest.fixture(scope="session")
+def run_prefix():
+    "A datetime string used to avoid stale data left over from previous tests"
+    return datetime.strftime(datetime.now(), "%Y%m%d%H%M%S")
+
+
 @pytest.fixture(scope="session")
 def xnat_repository():
     xnat4tests.start_xnat()
@@ -54,21 +57,35 @@ def xnat_archive_dir(xnat_repository):
     return xnat4tests.Config().xnat_root_dir / "archive"
 
 
+@pytest.fixture(scope="session")
+def tmp_gen_dir():
+    # tmp_gen_dir = Path("~").expanduser() / ".xnat-ingest-work3"
+    # tmp_gen_dir.mkdir(exist_ok=True)
+    # return tmp_gen_dir
+    return Path(tempfile.mkdtemp())
+
+
 @pytest.fixture(scope="session")
 def xnat_login(xnat_repository):
-    # Ensure that project ID is present in test XNAT before we connect, as new projects
-    # often don't show up until you log-off/log-in again
-    with xnat4tests.connect() as xlogin:
-        try:
-            xlogin.projects[PROJECT_ID]
-        except KeyError:
-            xlogin.put(f"/data/archive/projects/{PROJECT_ID}")
     return xnat4tests.connect()
 
 
 @pytest.fixture(scope="session")
-def xnat_server(xnat_repository):
-    return xnat4tests.Config().xnat_uri
+def xnat_project(xnat_login, run_prefix):
+    project_id = f"INGESTUPLOAD{run_prefix}"
+    with xnat4tests.connect() as xnat_login:
+        xnat_login.put(f"/data/archive/projects/{project_id}")
+    return project_id
+
+
+@pytest.fixture(scope="session")
+def xnat_server(xnat_config):
+    return xnat_config.xnat_uri
+
+
+@pytest.fixture(scope="session")
+def xnat_config(xnat_repository):
+    return xnat4tests.Config()
 
 
 @pytest.fixture
@@ -81,22 +98,6 @@ def invoke(*args, catch_exceptions=catch_cli_exceptions, **kwargs):
     return invoke
 
 
-@pytest.fixture
-def export_dir(tmp_path: Path) -> Path:
-    dicom_dir = get_image()
-    export_dir = tmp_path / "export-dir"
-    export_dir.mkdir()
-    session_dir = export_dir / "test-session"
-    shutil.copytree(dicom_dir, session_dir)
-    get_raw_data_files(session_dir)
-    return export_dir
-
-
-@pytest.fixture
-def xnat_project(xnat_login, scope="session"):
-    return xnat_login.projects[PROJECT_ID]
-
-
 # Create a custom handler that captures email messages for testing
 class TestSMTPHandler(SMTPHandler):
     def __init__(

pyproject.toml

@@ -7,7 +7,15 @@ name = "xnat-exported-scans"
 description = "Uploads exported DICOM and raw data to XNAT, parsing metadata from DICOMs"
 readme = "README.md"
 requires-python = ">=3.8"
-dependencies = ["click >=8.1", "pydicom >=2.3.1", "tqdm >=4.64.1", "xnat"]
+dependencies = [
+    "click >=8.1",
+    "fileformats-medimage-extras",
+    "pydicom >=2.3.1",
+    "tqdm >=4.64.1",
+    "xnat",
+    "arcana",
+    "arcana-xnat",
+]
 license = { file = "LICENSE" }
 authors = [{ name = "Thomas G. Close", email = "[email protected]" }]
 maintainers = [
@@ -45,24 +53,24 @@ tests = ["xnat-exported-scans[test]"]
 all = ["xnat-exported-scans[dev,test]"]
 
 [project.urls]
-repository = "https://github.com/Australian-Imaging-Service/xnat_exported_scans"
+repository = "https://github.com/Australian-Imaging-Service/xnat_ingest"
 
 [project.scripts]
-xnat-upload-exported-scans = "xnat_exported_scans.cli:cli"
+xnat-ingest = "xnat_ingest.cli:cli"
 
 [tool.hatch.version]
 source = "vcs"
 
 [tool.hatch.build.hooks.vcs]
-version-file = "xnat_exported_scans/_version.py"
+version-file = "xnat_ingest/_version.py"
 
 [tool.hatch.build]
-packages = ["xnat_exported_scans"]
+packages = ["xnat_ingest"]
 exclude = ["/tests"]
 
 [tool.black]
 target-version = ['py37', 'py38']
-exclude = "xnat_exported_scans/_version.py"
+exclude = "xnat_ingest/_version.py"
 
 [tool.codespell]
 ignore-words = ".codespell-ignorewords"