debugging unnitests after change to using frametree and switching to …

…flexible formats

pyproject.toml

@@ -16,8 +16,8 @@ dependencies = [
     "natsort",
     "paramiko",
     "xnat",
-    "arcana",
-    "arcana-xnat >=0.4.1",
+    "frametree",
+    "frametree-xnat",
 ]
 license = { file = "LICENSE" }
 authors = [{ name = "Thomas G. Close", email = "[email protected]" }]

scripts/dcm_performance_mrtrix.py

@@ -9,9 +9,9 @@
     "StudyInstanceUID",
     "StudyID",
     "PatientID",
-    "AccessionNumber"
+    "AccessionNumber",
 ]
 
-series = DicomSeries(get_image().iterdir())
+series = DicomSeries(get_image().iterdir(), specific_tags=METADATA_KEYS)
 
-timeit.timeit(lambda: series.select_metadata(METADATA_KEYS))
+timeit.timeit(lambda: series.metadata)

scripts/dcm_performance_pydicom.py

@@ -8,9 +8,9 @@
     "StudyInstanceUID",
     "StudyID",
     "PatientID",
-    "AccessionNumber"
+    "AccessionNumber",
 ]
 
-series = DicomSeries(get_image().iterdir())
+series = DicomSeries(get_image().iterdir(), specific_tags=METADATA_KEYS)
 
-timeit.timeit(lambda: series.select_metadata(METADATA_KEYS))
+timeit.timeit(lambda: series.metadata)

xnat_ingest/cli/stage.py

@@ -229,7 +229,7 @@ def stage(
     files_path: str,
     staging_dir: Path,
     datatype: str,
-    associated_files: AssociatedFiles,
+    associated_files: ty.List[AssociatedFiles],
     project_field: str,
     subject_field: str,
     visit_field: str,
@@ -274,10 +274,10 @@ def stage(
 
     msg = f"Loading {datatype} sessions from '{files_path}'"
 
-    if associated_files:
-        msg += f" with associated files selected from '{associated_files.glob}'"
-        if not associated_files.glob.startswith("/"):
-            msg += " (relative to the directories in which the DICOMs are found)"
+    for assoc_files in associated_files:
+        msg += f" with associated files selected from '{assoc_files.glob}'"
+        if not assoc_files.glob.startswith("/"):
+            msg += " (relative to the directories in which the primary files are found)"
 
     logger.info(msg)
 

xnat_ingest/cli/upload.py

@@ -11,13 +11,13 @@
 import click
 from tqdm import tqdm
 from natsort import natsorted
-import xnat
+import xnat  # type: ignore[import-untyped]
 import boto3
 import paramiko
 from fileformats.generic import File
-from arcana.core.data.set import Dataset
-from arcana.xnat import Xnat
-from xnat.exceptions import XNATResponseError
+from frametree.core.frameset import FrameSet  # type: ignore[import-untyped]
+from frametree.xnat import Xnat  # type: ignore[import-untyped]
+from xnat.exceptions import XNATResponseError  # type: ignore[import-untyped]
 from xnat_ingest.cli.base import cli
 from xnat_ingest.session import ImagingSession
 from xnat_ingest.utils import (
@@ -349,7 +349,7 @@ def iter_staged_sessions():
             missing_datasets = set()
             for project_id in project_ids:
                 try:
-                    dataset = Dataset.load(project_id, xnat_repo)
+                    dataset = FrameSet.load(project_id, xnat_repo)
                 except Exception:
                     missing_datasets.add(project_id)
                 else:
@@ -392,7 +392,7 @@ def iter_staged_sessions():
 
                 # Access Arcana dataset associated with project
                 try:
-                    dataset = Dataset.load(session.project_id, xnat_repo)
+                    dataset = FrameSet.load(session.project_id, xnat_repo)
                 except Exception as e:
                     logger.warning(
                         "Did not load dataset definition (%s) from %s project "

xnat_ingest/session.py

@@ -941,9 +941,9 @@ class MockStore(Store):
     @property
     def row(self):
         return DataRow(
-            ids={DummySpace._: None},
-            dataset=FrameSet(id=None, store=self, hierarchy=[], space=DummySpace),
-            frequency=DummySpace._,
+            ids={DummyAxes._: None},
+            frameset=FrameSet(id=None, store=self, hierarchy=[], axes=DummyAxes),
+            frequency=DummyAxes._,
         )
 
     def populate_row(self, row: DataRow):
@@ -1011,7 +1011,7 @@ def create_data_tree(
         id: str,
         leaves: ty.List[ty.Tuple[str, ...]],
         hierarchy: ty.List[str],
-        space: type,
+        axes: type,
         **kwargs,
     ):
         raise NotImplementedError
@@ -1029,12 +1029,12 @@ def put_provenance(self, provenance: ty.Dict[str, ty.Any], entry: DataEntry):
     def get_provenance(self, entry: DataEntry) -> ty.Dict[str, ty.Any]:
         raise NotImplementedError
 
-    def save_dataset_definition(
+    def save_frameset_definition(
         self, dataset_id: str, definition: ty.Dict[str, ty.Any], name: str
     ):
         raise NotImplementedError
 
-    def load_dataset_definition(
+    def load_frameset_definition(
         self, dataset_id: str, name: str
     ) -> ty.Dict[str, ty.Any]:
         raise NotImplementedError
@@ -1046,5 +1046,5 @@ def create_entry(self, path: str, datatype: type, row: DataRow) -> DataEntry:
         raise NotImplementedError
 
 
-class DummySpace(Axes):
+class DummyAxes(Axes):
     _ = 0b0

xnat_ingest/tests/test_cli.py

@@ -1,28 +1,28 @@
 import os
 import shutil
 from pathlib import Path
-from arcana.core.cli.dataset import (
+from frametree.core.cli import (  # type: ignore[import-untyped]
     define as dataset_define,
     add_source as dataset_add_source,
 )
-import xnat4tests
-from arcana.core.cli.store import add as store_add
+import xnat4tests  # type: ignore[import-untyped]
+from frametree.core.cli.store import add as store_add  # type: ignore[import-untyped]
 from xnat_ingest.cli import stage, upload
 from xnat_ingest.utils import show_cli_trace
 from fileformats.medimage import DicomSeries
-from medimages4tests.dummy.dicom.pet.wholebody.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.dicom.pet.wholebody.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_pet_image,
 )
-from medimages4tests.dummy.dicom.ct.ac.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.dicom.ct.ac.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_ac_image,
 )
-from medimages4tests.dummy.dicom.pet.topogram.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.dicom.pet.topogram.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_topogram_image,
 )
-from medimages4tests.dummy.dicom.pet.statistics.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.dicom.pet.statistics.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_statistics_image,
 )
-from medimages4tests.dummy.raw.pet.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.raw.pet.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_files as get_raw_data_files,
 )
 
@@ -136,8 +136,8 @@ def test_stage_and_upload(
     result = cli_runner(
         store_add,
         [
+            "xnat",
             "testxnat",
-            "xnat:Xnat",
             "--server",
             xnat_server,
             "--user",

xnat_ingest/tests/test_dicom.py

@@ -1,22 +1,23 @@
 import pytest
 import platform
-from medimages4tests.dummy.dicom.pet.wholebody.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.dicom.pet.wholebody.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_pet_image,
 )
 from fileformats.medimage import DicomSeries
-from xnat_ingest.session import ImagingSession
 
 # PATIENT_ID = "patient-id"
 # STUDY_ID = "study-id"
 # ACCESSION_NUMBER = "accession-number"
 
 
 @pytest.fixture
-def dicom_series(scope="module") -> ImagingSession:
+def dicom_series(scope="module") -> DicomSeries:
     return DicomSeries(get_pet_image().iterdir())
 
 
-@pytest.mark.xfail(condition=(platform.system() == "Linux"), reason="Not working on ubuntu")
+@pytest.mark.xfail(
+    condition=(platform.system() == "Linux"), reason="Not working on ubuntu"
+)
 def test_mrtrix_dicom_metadata(dicom_series: DicomSeries):
     keys = [
         "AccessionNumber",
@@ -26,13 +27,15 @@ def test_mrtrix_dicom_metadata(dicom_series: DicomSeries):
         "StudyInstanceUID",
         "SOPInstanceUID",
     ]
-    dicom_series.select_metadata(keys)
+    dicom_series = DicomSeries(dicom_series, specific_tags=keys)
 
-    assert sorted(dicom_series.metadata) == sorted(keys + ['SpecificCharacterSet'])
+    assert sorted(dicom_series.metadata) == sorted(keys + ["SpecificCharacterSet"])
     assert dicom_series.metadata["PatientName"] == "GivenName^FamilyName"
     assert dicom_series.metadata["AccessionNumber"] == "987654321"
-    assert dicom_series.metadata["PatientID"] == 'Session Label'
+    assert dicom_series.metadata["PatientID"] == "Session Label"
     assert dicom_series.metadata["StudyID"] == "PROJECT_ID"
     assert not isinstance(dicom_series.metadata["StudyInstanceUID"], list)
     assert isinstance(dicom_series.metadata["SOPInstanceUID"], list)
-    assert len(dicom_series.metadata["SOPInstanceUID"]) == len(list(dicom_series.contents))
+    assert len(dicom_series.metadata["SOPInstanceUID"]) == len(
+        list(dicom_series.contents)
+    )

xnat_ingest/tests/test_session.py

@@ -8,29 +8,29 @@
     Vnd_Siemens_Biograph128Vision_Vr20b_PetListMode,
     Vnd_Siemens_Biograph128Vision_Vr20b_PetSinogram,
 )
-from arcana.core.data.set import Dataset
-from arcana.common import DirTree
-from medimages4tests.dummy.dicom.base import default_dicom_dir
-from medimages4tests.dummy.dicom.pet.wholebody.siemens.biograph_vision.vr20b import (
+from frametree.core.frameset import FrameSet  # type: ignore[import-untyped]
+from frametree.common import FileSystem  # type: ignore[import-untyped]
+from medimages4tests.dummy.dicom.base import default_dicom_dir  # type: ignore[import-untyped]
+from medimages4tests.dummy.dicom.pet.wholebody.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_pet_image,
     __file__ as pet_src_file,
 )
-from medimages4tests.dummy.dicom.ct.ac.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.dicom.ct.ac.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_ac_image,
     __file__ as ac_src_file,
 )
-from medimages4tests.dummy.dicom.pet.topogram.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.dicom.pet.topogram.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_topogram_image,
     __file__ as topogram_src_file,
 )
-from medimages4tests.dummy.dicom.pet.statistics.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.dicom.pet.statistics.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_image as get_statistics_image,
     __file__ as statistics_src_file,
 )
-from medimages4tests.dummy.raw.pet.siemens.biograph_vision.vr20b import (
+from medimages4tests.dummy.raw.pet.siemens.biograph_vision.vr20b import (  # type: ignore[import-untyped]
     get_files as get_raw_data_files,
 )
-from xnat_ingest.session import ImagingSession, ImagingScan, DummySpace
+from xnat_ingest.session import ImagingSession, ImagingScan, DummyAxes
 from xnat_ingest.utils import AssociatedFiles
 
 
@@ -84,7 +84,7 @@ def imaging_session() -> ImagingSession:
 
 
 @pytest.fixture
-def dataset(tmp_path: Path) -> Dataset:
+def dataset(tmp_path: Path) -> FrameSet:
     """For use in tests, this method creates a test dataset from the provided
     blueprint
 
@@ -104,12 +104,12 @@ def dataset(tmp_path: Path) -> Dataset:
         passed through to create_dataset
     """
     dataset_path = tmp_path / "a-dataset"
-    store = DirTree()
+    store = FileSystem()
     dataset = store.create_dataset(
         id=dataset_path,
         leaves=[],
         hierarchy=[],
-        space=DummySpace,
+        axes=DummyAxes,
     )
     for col_name, col_type, col_pattern in [
         ("pet", "medimage/dicom-series", "PET SWB 8MIN"),
@@ -139,7 +139,7 @@ def dataset(tmp_path: Path) -> Dataset:
     condition=platform.system() == "Linux", reason="Not working on ubuntu"
 )
 def test_session_select_resources(
-    imaging_session: ImagingSession, dataset: Dataset, tmp_path: Path
+    imaging_session: ImagingSession, dataset: FrameSet, tmp_path: Path
 ):
 
     assoc_dir = tmp_path / "assoc"