diff --git a/pyproject.toml b/pyproject.toml index 20756a9..5ecd3e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "aind-smartspim-data-transformation" description = "Generated from aind-library-template" license = {text = "MIT"} -requires-python = ">=3.9" +requires-python = ">=3.9.2" authors = [ {name = "Allen Institute for Neural Dynamics"} ] @@ -49,7 +49,7 @@ version = {attr = "aind_smartspim_data_transformation.__version__"} [tool.black] line-length = 79 -target_version = ['py36'] +target_version = ['py39'] exclude = ''' ( diff --git a/scripts/singularity_build.def b/scripts/singularity_build.def index cc05c6b..415a227 100644 --- a/scripts/singularity_build.def +++ b/scripts/singularity_build.def @@ -6,17 +6,10 @@ From: python:3.10-bullseye cp -R . ${SINGULARITY_ROOTFS}/aind-smartspim-data-transformation %post - # Installing dask mpi - wget https://www.mpich.org/static/downloads/3.2/mpich-3.2.tar.gz - tar xfz mpich-3.2.tar.gz - rm mpich-3.2.tar.gz - mkdir mpich-build - cd mpich-build - ../mpich-3.2/configure --disable-fortran 2>&1 | tee c.txt - make 2>&1 | tee m.txt - make install 2>&1 | tee mi.txt - cd .. - + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + rm awscliv2.zip + ./aws/install cd ${SINGULARITY_ROOTFS}/aind-smartspim-data-transformation - pip install . mpi4py dask_mpi --no-cache-dir + pip install . --no-cache-dir rm -rf ${SINGULARITY_ROOTFS}/aind-smartspim-data-transformation diff --git a/src/aind_smartspim_data_transformation/_shared/__init__.py b/src/aind_smartspim_data_transformation/_shared/__init__.py deleted file mode 100644 index 8755617..0000000 --- a/src/aind_smartspim_data_transformation/_shared/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Shared constants -""" diff --git a/src/aind_smartspim_data_transformation/_shared/types.py b/src/aind_smartspim_data_transformation/_shared/types.py deleted file mode 100644 index 8371ae8..0000000 --- a/src/aind_smartspim_data_transformation/_shared/types.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -Defines all the types used in the module -""" - -from pathlib import Path -from typing import Union - -import dask.array as da -import numpy as np - -PathLike = Union[Path, str] -ArrayLike = Union[da.Array, np.ndarray] diff --git a/src/aind_smartspim_data_transformation/compress/dask_utils.py b/src/aind_smartspim_data_transformation/compress/dask_utils.py deleted file mode 100644 index 1e555be..0000000 --- a/src/aind_smartspim_data_transformation/compress/dask_utils.py +++ /dev/null @@ -1,174 +0,0 @@ -""" -Module for dask utilities -""" - -import logging -import os -import socket -from enum import Enum -from typing import Optional, Tuple - -import distributed -import requests -from distributed import Client, LocalCluster - -try: - from dask_mpi import initialize - - DASK_MPI_INSTALLED = True -except ImportError: - DASK_MPI_INSTALLED = False - -LOGGER = logging.getLogger(__name__) - - -class Deployment(Enum): - """Deployment enums""" - - LOCAL = "local" - SLURM = "slurm" - - -def log_dashboard_address( - client: distributed.Client, login_node_address: str = "hpc-login" -) -> None: - """ - Logs the terminal command required to access the Dask dashboard - - Args: - client: the Client instance - login_node_address: the address of the cluster login node - """ - host = client.run_on_scheduler(socket.gethostname) # noqa: F841 - port = client.scheduler_info()["services"]["dashboard"] # noqa: F841 - user = os.getenv("USER") # noqa: F841 - LOGGER.info( - f"To access the dashboard, run the following in " - f"a terminal: ssh -L {port}:{host}:{port} {user}@" - f"{login_node_address} " - ) - - -def get_deployment() -> str: - """ - Gets the SLURM deployment if this - exists - - Returns - ------- - str - SLURM_JOBID - """ - if os.getenv("SLURM_JOBID") is None: - deployment = Deployment.LOCAL.value - else: - # we're running on the Allen HPC - deployment = Deployment.SLURM.value - return deployment - - -def get_client( - deployment: str = Deployment.LOCAL.value, - worker_options: Optional[dict] = None, - n_workers: int = 1, - processes=True, -) -> Tuple[distributed.Client, int]: - """ - Create a distributed Client - - Args: - deployment: the type of deployment. Either "local" or "slurm" - worker_options: a dictionary of options to pass to the worker class - n_workers: the number of workers (only applies to "local" deployment) - - Returns: - the distributed Client and number of workers - """ - if deployment == Deployment.SLURM.value: - if not DASK_MPI_INSTALLED: - raise ImportError( - "dask-mpi must be installed to use the SLURM deployment" - ) - if worker_options is None: - worker_options = {} - slurm_job_id = os.getenv("SLURM_JOBID") - if slurm_job_id is None: - raise Exception( - "SLURM_JOBID environment variable is not set." - "Are you running under SLURM?" - ) - initialize( - nthreads=int(os.getenv("SLURM_CPUS_PER_TASK", 1)), - local_directory=f"/scratch/fast/{slurm_job_id}", - worker_class="distributed.nanny.Nanny", - worker_options=worker_options, - ) - client = Client() - log_dashboard_address(client) - n_workers = int(os.getenv("SLURM_NTASKS")) - elif deployment == Deployment.LOCAL.value: - client = Client( - LocalCluster( - n_workers=n_workers, processes=processes, threads_per_worker=1 - ) - ) - else: - raise NotImplementedError - return client, n_workers - - -def cancel_slurm_job( - job_id: str, api_url: str, headers: dict -) -> requests.Response: - """ - Attempt to release resources and cancel the job - - Args: - job_id: the SLURM job ID - api_url: the URL of the SLURM REST API. - E.g., "http://myhost:80/api/slurm/v0.0.36" - - Raises: - HTTPError: if the request to cancel the job fails - """ - # Attempt to release resources and cancel the job - # Workaround for https://github.com/dask/dask-mpi/issues/87 - endpoint = f"{api_url}/job/{job_id}" - response = requests.delete(endpoint, headers=headers) - - return response - - -def _cleanup(deployment: str) -> None: - """ - Clean up any resources that were created during the job. - - Parameters - ---------- - deployment : str - The type of deployment. Either "local" or "slurm" - """ - if deployment == Deployment.SLURM.value: - job_id = os.getenv("SLURM_JOBID") - if job_id is not None: - try: - api_url = f"http://{os.environ['HPC_HOST']}" - api_url += f":{os.environ['HPC_PORT']}" - api_url += f"/{os.environ['HPC_API_ENDPOINT']}" - headers = { - "X-SLURM-USER-NAME": os.environ["HPC_USERNAME"], - "X-SLURM-USER-PASSWORD": os.environ["HPC_PASSWORD"], - "X-SLURM-USER-TOKEN": os.environ["HPC_TOKEN"], - } - except KeyError as ke: - logging.error(f"Failed to get SLURM env vars to cleanup: {ke}") - return - logging.info(f"Cancelling SLURM job {job_id}") - response = cancel_slurm_job(job_id, api_url, headers) - if response.status_code != 200: - logging.error( - f"Failed to cancel SLURM job {job_id}: {response.text}" - ) - else: - # This might not run if the job is cancelled - logging.info(f"Cancelled SLURM job {job_id}") diff --git a/src/aind_smartspim_data_transformation/compress/png_to_zarr.py b/src/aind_smartspim_data_transformation/compress/png_to_zarr.py index cace58c..e687655 100644 --- a/src/aind_smartspim_data_transformation/compress/png_to_zarr.py +++ b/src/aind_smartspim_data_transformation/compress/png_to_zarr.py @@ -6,11 +6,8 @@ """ import logging -import multiprocessing import os import time -from datetime import datetime -from pathlib import Path from typing import Dict, Hashable, List, Optional, Sequence, Tuple, Union, cast import dask @@ -19,52 +16,19 @@ import pims import xarray_multiscale import zarr -from dask import config as da_cfg from dask.array.core import Array from dask.base import tokenize -from dask.distributed import Client, LocalCluster, performance_report - -# from distributed import wait from numcodecs import blosc from ome_zarr.format import CurrentFormat from ome_zarr.io import parse_url from ome_zarr.writer import write_multiscales_metadata from skimage.io import imread as sk_imread -from aind_smartspim_data_transformation._shared.types import ( - ArrayLike, - PathLike, -) from aind_smartspim_data_transformation.compress.zarr_writer import ( BlockedArrayWriter, ) -from aind_smartspim_data_transformation.io import PngReader from aind_smartspim_data_transformation.io.utils import pad_array_n_d - - -def set_dask_config(dask_folder: str): - """ - Sets dask configuration - - Parameters - ---------- - dask_folder: str - Path to the temporary directory and local directory - of workers in dask. - """ - # Setting dask configuration - da_cfg.set( - { - "temporary-directory": dask_folder, - "local_directory": dask_folder, - # "tcp-timeout": "300s", - "array.chunk-size": "128MiB", - "distributed.worker.memory.target": 0.90, # 0.85, - "distributed.worker.memory.spill": 0.92, # False,# - "distributed.worker.memory.pause": 0.95, # False,# - "distributed.worker.memory.terminate": 0.98, - } - ) +from aind_smartspim_data_transformation.models import ArrayLike, PathLike def _build_ome( @@ -587,6 +551,7 @@ def smartspim_channel_zarr_writer( Logger object """ + # Getting channel color tmp_channel_name = channel_name.replace(".zarr", "") em_wav = int(tmp_channel_name.split("_")[-1]) @@ -596,7 +561,9 @@ def smartspim_channel_zarr_writer( image_data = image_data.rechunk(final_chunksize) image_data = pad_array_n_d(arr=image_data) - print(f"Writing {image_data} from {stack_name} in {output_path}") + image_name = stack_name + + print(f"Writing {image_data} from {stack_name} to {output_path}") # Creating Zarr dataset store = parse_url(path=output_path, mode="w").store @@ -628,14 +595,14 @@ def smartspim_channel_zarr_writer( channel_startend = [(0.0, 350.0) for _ in range(image_data.shape[1])] new_channel_group = root_group.create_group( - name=stack_name, overwrite=True + name=image_name, overwrite=True ) # Writing OME-NGFF metadata write_ome_ngff_metadata( group=new_channel_group, arr=image_data, - image_name=channel_name, + image_name=image_name, n_lvls=n_lvls, scale_factors=scale_factor, voxel_size=voxel_size, @@ -646,198 +613,64 @@ def smartspim_channel_zarr_writer( metadata=_get_pyramid_metadata(), ) - performance_report_path = f"{output_path}/report_{stack_name}.html" + # performance_report_path = f"{output_path}/report_{stack_name}.html" start_time = time.time() # Writing zarr and performance report - with performance_report(filename=performance_report_path): - logger.info(f"{'='*40}Writing channel {channel_name}{'='*40}") + # with performance_report(filename=performance_report_path): + logger.info(f"Writing channel {channel_name}/{stack_name}") - # Writing zarr - block_shape = list( - BlockedArrayWriter.get_block_shape( - arr=image_data, target_size_mb=12800 # 51200, - ) + # Writing zarr + block_shape = list( + BlockedArrayWriter.get_block_shape( + arr=image_data, target_size_mb=12800 # 51200, ) + ) - # Formatting to 5D block shape - block_shape = ([1] * (5 - len(block_shape))) + block_shape - written_pyramid = [] - pyramid_group = None - - # Writing multiple levels - for level in range(n_lvls): - if not level: - array_to_write = image_data - - else: - # It's faster to write the scale and then read it back - # to compute the next scale - previous_scale = da.from_zarr( - pyramid_group, pyramid_group.chunks - ) - new_scale_factor = ( - [1] * (len(previous_scale.shape) - len(scale_factor)) - ) + scale_factor - - previous_scale_pyramid, _ = compute_pyramid( - data=previous_scale, - scale_axis=new_scale_factor, - chunks=image_data.chunksize, - n_lvls=2, - ) - array_to_write = previous_scale_pyramid[-1] - - logger.info(f"[level {level}]: pyramid level: {array_to_write}") - - # Create the scale dataset - pyramid_group = new_channel_group.create_dataset( - name=level, - shape=array_to_write.shape, - chunks=array_to_write.chunksize, - dtype=array_to_write.dtype, - compressor=writing_options, - dimension_separator="/", - overwrite=True, + # Formatting to 5D block shape + block_shape = ([1] * (5 - len(block_shape))) + block_shape + written_pyramid = [] + pyramid_group = None + + # Writing multiple levels + for level in range(n_lvls): + if not level: + array_to_write = image_data + + else: + # It's faster to write the scale and then read it back + # to compute the next scale + previous_scale = da.from_zarr(pyramid_group, pyramid_group.chunks) + new_scale_factor = ( + [1] * (len(previous_scale.shape) - len(scale_factor)) + ) + scale_factor + + previous_scale_pyramid, _ = compute_pyramid( + data=previous_scale, + scale_axis=new_scale_factor, + chunks=image_data.chunksize, + n_lvls=2, ) + array_to_write = previous_scale_pyramid[-1] + + logger.info(f"[level {level}]: pyramid level: {array_to_write}") + + # Create the scale dataset + pyramid_group = new_channel_group.create_dataset( + name=level, + shape=array_to_write.shape, + chunks=array_to_write.chunksize, + dtype=array_to_write.dtype, + compressor=writing_options, + dimension_separator="/", + overwrite=True, + ) - # Block Zarr Writer - BlockedArrayWriter.store( - array_to_write, pyramid_group, block_shape - ) - written_pyramid.append(array_to_write) + # Block Zarr Writer + BlockedArrayWriter.store(array_to_write, pyramid_group, block_shape) + written_pyramid.append(array_to_write) end_time = time.time() logger.info(f"Time to write the dataset: {end_time - start_time}") print(f"Time to write the dataset: {end_time - start_time}") logger.info(f"Written pyramid: {written_pyramid}") - - -def convert_stacks_to_ome_zarr(channel_path, logger, output_path): - """ - Converts image stacks from PNG to OMEZarr. - - Parameters - ---------- - channel_path: str - Path where the stacks for the channel - are located. - - logger: logging.Logger - Logging object - - output_path: str - Path where we want to write the converted - stacks to OMEZarr. - - """ - # channel_path must end with Ex_{wav}_Em_{wav} - - # Setting up local cluster - n_workers = multiprocessing.cpu_count() - logger.info(f"Setting {n_workers} workers") - threads_per_worker = 1 - - # Instantiating local cluster for parallel writing - cluster = LocalCluster( - n_workers=n_workers, - threads_per_worker=threads_per_worker, - processes=True, - memory_limit="auto", - ) - - client = Client(cluster) - - cols = os.listdir(channel_path) - for col in cols: - curr_col = channel_path.joinpath(col) - for row in os.listdir(curr_col): - curr_row = curr_col.joinpath(row) - delayed_stack = PngReader( - data_path=f"{curr_row}/*.png" - ).as_dask_array() - print(f"Writing curr stack {curr_row}: {delayed_stack}") - - smartspim_channel_zarr_writer( - image_data=delayed_stack, - output_path=Path(output_path).joinpath(f"{channel_path.stem}"), - voxel_size=[2.0, 1.8, 1.8], - final_chunksize=(128, 128, 128), - scale_factor=[2, 2, 2], - codec="zstd", - compression_level=0, - n_lvls=4, - channel_name=channel_path.stem, - logger=logger, - stack_name=f"{col}_{row.split('_')[-1]}.zarr", - client=client, - ) - - -def create_logger(output_log_path: PathLike) -> logging.Logger: - """ - Creates a logger that generates - output logs to a specific path. - - Parameters - ------------ - output_log_path: PathLike - Path where the log is going - to be stored - - Returns - ----------- - logging.Logger - Created logger pointing to - the file path. - """ - CURR_DATE_TIME = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - LOGS_FILE = f"{output_log_path}/fusion_log_{CURR_DATE_TIME}.log" - - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s : %(message)s", - datefmt="%Y-%m-%d %H:%M", - handlers=[ - logging.StreamHandler(), - logging.FileHandler(LOGS_FILE, "a"), - ], - force=True, - ) - - logging.disable("DEBUG") - logger = logging.getLogger(__name__) - logger.setLevel(logging.DEBUG) - - return logger - - -def main(): - """ - Main function in how to use these functions - """ - - data_folder = Path(os.path.abspath("../data")) - results_folder = Path(os.path.abspath("../results")) - scratch_folder = Path(os.path.abspath("../scratch")) - - set_dask_config(dask_folder=scratch_folder) - - logger = create_logger(output_log_path=results_folder) - - BASE_PATH = data_folder.joinpath( - "SmartSPIM_692911_2023-10-23_11-27-30/SmartSPIM" - ) - - channels = os.listdir(str(BASE_PATH)) - - for channel in channels: - convert_stacks_to_ome_zarr( - channel_path=BASE_PATH.joinpath(channel), - logger=logger, - output_path=results_folder, - ) - - -if __name__ == "__main__": - main() diff --git a/src/aind_smartspim_data_transformation/compress/zarr_writer.py b/src/aind_smartspim_data_transformation/compress/zarr_writer.py index 7041c9b..254ecdc 100644 --- a/src/aind_smartspim_data_transformation/compress/zarr_writer.py +++ b/src/aind_smartspim_data_transformation/compress/zarr_writer.py @@ -48,7 +48,7 @@ def _closer_to_target( return shape2 -def expand_chunks( +def expand_chunks( # noqa: C901 chunks: Tuple[int, int, int], data_shape: Tuple[int, int, int], target_size: int, diff --git a/src/aind_smartspim_data_transformation/io/__init__.py b/src/aind_smartspim_data_transformation/io/__init__.py index baff90e..4922259 100644 --- a/src/aind_smartspim_data_transformation/io/__init__.py +++ b/src/aind_smartspim_data_transformation/io/__init__.py @@ -1,7 +1,3 @@ """ Input and output operations """ - -# flake8: noqa: F403 -from ._io import * -from .utils import * diff --git a/src/aind_smartspim_data_transformation/io/_io.py b/src/aind_smartspim_data_transformation/io/_io.py deleted file mode 100644 index 6effab0..0000000 --- a/src/aind_smartspim_data_transformation/io/_io.py +++ /dev/null @@ -1,634 +0,0 @@ -""" -Module that defines base Image Reader class -and the available metrics -""" - -import os -from abc import ABC, abstractmethod, abstractproperty -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union - -import dask.array as da -import imageio as iio -import numpy as np -import pims -import tifffile -import zarr -from dask.array.core import Array -from dask.base import tokenize -from dask_image.imread import imread as daimread -from skimage.io import imread as sk_imread - -from .utils import add_leading_dim, read_json_as_dict - -# IO types -PathLike = Union[str, Path] -ArrayLike = Union[da.Array, np.ndarray] - - -class ImageReader(ABC): - """ - Abstract class to create image readers - classes - """ - - def __init__(self, data_path: PathLike) -> None: - """ - Class constructor of image reader. - - Parameters - ------------------------ - data_path: PathLike - Path where the image is located - - """ - - self.__data_path = data_path - super().__init__() - - @abstractmethod - def as_dask_array(self, chunk_size: Optional[Any] = None) -> da.Array: - """ - Abstract method to return the image as a dask array. - - Parameters - ------------------------ - chunk_size: Optional[Any] - If provided, the image will be rechunked to the desired - chunksize - - Returns - ------------------------ - da.Array - Dask array with the image - - """ - pass - - @abstractmethod - def as_numpy_array(self) -> np.ndarray: - """ - Abstract method to return the image as a numpy array. - - Returns - ------------------------ - np.ndarray - Numpy array with the image - - """ - pass - - @abstractmethod - def metadata(self) -> Dict: - """ - Abstract method that return the image metadata. - - Returns - ------- - Dict - Dictionary with image metadata - """ - pass - - @abstractmethod - def close_handler(self) -> None: - """ - Abstract method to close the image hander when it's necessary. - - """ - pass - - @abstractmethod - def indexing(self, xv: np.array, yv: np.array) -> ArrayLike: - """ - Abstract method to index arrays - """ - pass - - @abstractproperty - def shape(self) -> Tuple: - """ - Abstract method to return the shape of the image. - - Returns - ------------------------ - Tuple - Tuple with the shape of the image - - """ - pass - - @abstractproperty - def chunks(self) -> Tuple: - """ - Abstract method to return the chunks of the image if it's possible. - - Returns - ------------------------ - Tuple - Tuple with the chunks of the image - - """ - pass - - @property - def data_path(self) -> PathLike: - """ - Getter to return the path where the image is located. - - Returns - ------------------------ - PathLike - Path of the image - - """ - return self.__data_path - - @data_path.setter - def data_path(self, new_data_path: PathLike) -> None: - """ - Setter of the path attribute where the image is located. - - Parameters - ------------------------ - new_data_path: PathLike - New path of the image - - """ - self.__data_path = new_data_path - - -class OMEZarrReader(ImageReader): - """ - OMEZarr reader class - """ - - def __init__( - self, - data_path: PathLike, - multiscale: Optional[str] = "0", - ) -> None: - """ - Class constructor of image OMEZarr reader. - - Parameters - ------------------------ - data_path: PathLike - Path where the image is located - - multiscale: Optional[str] - Desired multiscale to read from the image. Default: "0" - which is supposed to be the highest resolution - - """ - - # Adding multiscale to path - if isinstance(data_path, str): - data_path = f"{data_path}/{multiscale}" - - else: - data_path = data_path.joinpath(str(multiscale)) - - super().__init__(data_path=data_path) - self.lazy_image = da.from_zarr(self.data_path) - - def indexing(self, xv: np.array, yv: np.array) -> ArrayLike: - """ - Indexes arrays using X and Y locations - generated by a meshgrid - - Parameters - ---------- - xv: np.array - X locations generated by a meshgrid - - yv: np.array - y locations generated by a meshgrid - - Returns - ---------- - Data in provided locations - """ - - return self.lazy_image.vindex(xv, yv) - - def as_dask_array(self, chunk_size: Optional[Any] = None) -> da.array: - """ - Method to return the image as a dask array. - - Parameters - ------------------------ - chunk_size: Optional[Any] - If provided, the image will be rechunked to the desired - chunksize - - Returns - ------------------------ - da.Array - Dask array with the image - - """ - - if chunk_size: - return self.lazy_image.rechunk(chunks=chunk_size) - - return self.lazy_image - - def as_numpy_array(self): - """ - Method to return the image as a numpy array. - - Returns - ------------------------ - np.ndarray - Numpy array with the image - - """ - return zarr.open(self.data_path, "r")[:] - - def metadata(self) -> Dict: - """ - Returns the image metadata. - - Returns - ------- - Dict - Dictionary with image metadata - """ - metadata = {} - # Removing multiscale to path - if isinstance(self.data_path, str): - data_path = Path(self.data_path) - - zattrs_metadata = "" - zarray_metadata = "" - # Checking inside and outside of folder - # due to dimension separator "." or "/" - for path in [data_path, data_path.parent]: - if path.joinpath(".zattrs").exists(): - zattrs_metadata = path.joinpath(".zattrs") - - if path.joinpath(".zarray").exists(): - zarray_metadata = path.joinpath(".zarray") - - print(f"Reading metadata from {zattrs_metadata} and {zarray_metadata}") - - metadata[".zattrs"] = read_json_as_dict(zattrs_metadata) - metadata[".zarray"] = read_json_as_dict(zarray_metadata) - - return metadata - - def close_handler(self) -> None: - """ - Method to close the image hander when it's necessary. - """ - pass - - @property - def shape(self): - """ - Method to return the shape of the image. - - Returns - ------------------------ - Tuple - Tuple with the shape of the image - - """ - return zarr.open(self.data_path, "r").shape - - @property - def chunks(self): - """ - Method to return the chunks of the image. - - Returns - ------------------------ - Tuple - Tuple with the chunks of the image - - """ - return zarr.open(self.data_path, "r").chunks - - -class TiffReader(ImageReader): - """ - TiffReader class - """ - - def __init__(self, data_path: PathLike) -> None: - """ - Class constructor of image Tiff reader. - - Parameters - ------------------------ - data_path: PathLike - Path where the image is located - - """ - super().__init__(data_path) - self.tiff = tifffile.TiffFile(self.data_path) - - def indexing(self, xv: np.array, yv: np.array) -> ArrayLike: - """ - Indexes arrays using X and Y locations - generated by a meshgrid - - Parameters - ---------- - xv: np.array - X locations generated by a meshgrid - - yv: np.array - y locations generated by a meshgrid - - Returns - ---------- - Data in provided locations - """ - - return self.tiff.asarray()[xv, yv] - - def as_dask_array( - self, shape: Optional[Any] = None, dtype: Optional[Any] = None - ) -> da.Array: - """ - Method to return the image as a dask array. - - Parameters - ------------------------ - shape: Optional[Any] - Shape of the image - - Returns - ------------------------ - da.Array - Dask array with the image - - """ - data_path = str(self.data_path) - name = "imread-%s" % tokenize( - data_path, map(os.path.getmtime, data_path) - ) - - if shape is None or dtype is None: - with pims.open(data_path) as imgs: - shape = (1,) + (len(imgs),) + imgs.frame_shape - dtype = np.dtype(imgs.pixel_type) - - key = [(name,) + (0,) * len(shape)] - value = [(add_leading_dim, (sk_imread, data_path))] - dask_arr = dict(zip(key, value)) - chunk_size = tuple((d,) for d in shape) - - return Array(dask_arr, name, chunk_size, dtype) - - def metadata(self) -> Dict: - """ - Returns the image metadata. - - Returns - ------- - Dict - Dictionary with image metadata - """ - metadata = {} - with pims.open(self.data_path) as imgs: - metadata["shape"] = (1,) + (len(imgs),) + imgs.frame_shape - metadata["dtype"] = np.dtype(imgs.pixel_type) - - return metadata - - def as_numpy_array(self) -> np.ndarray: - """ - Abstract method to return the image as a numpy array. - - Returns - ------------------------ - np.ndarray - Numpy array with the image - - """ - return self.tiff.asarray() - - @property - def shape(self) -> Tuple: - """ - Abstract method to return the shape of the image. - - Returns - ------------------------ - Tuple - Tuple with the shape of the image - - """ - with pims.open(str(self.data_path)) as imgs: - shape = (len(imgs),) + imgs.frame_shape - - return shape - - @property - def chunks(self) -> Tuple: - """ - Abstract method to return the chunks of the image if it's possible. - - Returns - ------------------------ - Tuple - Tuple with the chunks of the image - - """ - return self.tiff.aszarr().chunks - - def close_handler(self) -> None: - """ - Closes image handler - """ - if self.tiff is not None: - self.tiff.close() - self.tiff = None - - def __del__(self) -> None: - """Overriding destructor to safely close image""" - self.close_handler() - - -class PngReader(ImageReader): - """ - PngReader class - """ - - def __init__(self, data_path: PathLike) -> None: - """ - Class constructor of image PNG reader. - - Parameters - ------------------------ - data_path: PathLike - Path where the image is located - - """ - super().__init__(data_path) - - def indexing(self, xv: np.array, yv: np.array) -> ArrayLike: - """ - Indexes arrays using X and Y locations - generated by a meshgrid - - Parameters - ---------- - xv: np.array - X locations generated by a meshgrid - - yv: np.array - y locations generated by a meshgrid - - Returns - ---------- - Data in provided locations - """ - return self.as_numpy_array[xv, yv] - - def as_dask_array(self, chunk_size: Optional[Any] = None) -> da.Array: - """ - Method to return the image as a dask array. - - Parameters - ------------------------ - chunk_size: Optional[Any] - If provided, the image will be rechunked to the desired - chunksize - - Returns - ------------------------ - da.Array - Dask array with the image - - """ - return daimread(self.data_path, arraytype="numpy") - - def as_numpy_array(self) -> np.ndarray: - """ - Abstract method to return the image as a numpy array. - - Returns - ------------------------ - np.ndarray - Numpy array with the image - - """ - return np.array(iio.imread(self.data_path)) - - @property - def shape(self) -> Tuple: - """ - Abstract method to return the shape of the image. - - Returns - ------------------------ - Tuple - Tuple with the shape of the image - - """ - with pims.open(str(self.data_path)) as imgs: - shape = (len(imgs),) + imgs.frame_shape - - return shape - - def metadata(self) -> Dict: - """ - Returns the image metadata. - - Returns - ------- - Dict - Dictionary with image metadata - """ - metadata = {} - with pims.open(self.data_path) as imgs: - metadata["shape"] = (len(imgs),) + imgs.frame_shape - - return metadata - - @property - def chunks(self) -> Tuple: - """ - Abstract method to return the chunks of the image if it's possible. - - Returns - ------------------------ - Tuple - Tuple with the chunks of the image - - """ - return self.as_dask_array().chunksize - - def close_handler(self) -> None: - """ - Closes image handler - """ - pass - - def __del__(self) -> None: - """Overriding destructor to safely close image""" - self.close_handler() - - -class ImageReaderFactory: - """ - Image reader factory class - """ - - def __init__(self): - """ - Class to create the image reader factory. - """ - self.__extensions = [".zarr", ".tif", ".tiff", ".png"] - self.factory = { - ".zarr": OMEZarrReader, - ".tif": TiffReader, - ".tiff": TiffReader, - ".png": PngReader, - } - - @property - def extensions(self) -> List: - """ - Method to return the allowed format extensions of the images. - - Returns - ------------------------ - List - List with the allowed image format extensions - - """ - return self.__extensions - - def create( - self, data_path: PathLike, parse_path: Optional[bool] = True, **kwargs - ) -> ImageReader: - """ - Method to create the image reader based on the format. - - Parameters - ---------- - data_path: PathLike - Path where the data is located - - parse_path: Optional[bool] - If True, parses the path with the pathlib.Path object. - Not useful when we're trying to access data in S3. - - Returns - ------- - List - List with the allowed image format extensions - - """ - path_cast = Path(data_path) if parse_path else data_path - ext = Path(data_path).suffix - - if ext not in self.__extensions: - raise NotImplementedError(f"File type {ext} not supported") - - return self.factory[ext](path_cast, **kwargs) diff --git a/src/aind_smartspim_data_transformation/io/readers.py b/src/aind_smartspim_data_transformation/io/readers.py new file mode 100644 index 0000000..0adb0ae --- /dev/null +++ b/src/aind_smartspim_data_transformation/io/readers.py @@ -0,0 +1,184 @@ +""" +Module that defines base Image Reader class +and the available metrics +""" + +from abc import ABC, abstractmethod +from typing import Any, Optional, Tuple + +import dask.array as da +import pims +from dask_image.imread import imread as daimread + +from aind_smartspim_data_transformation.models import PathLike + + +class ImageReader(ABC): + """ + Abstract class to create image readers + classes + """ + + def __init__(self, data_path: PathLike) -> None: + """ + Class constructor of image reader. + + Parameters + ------------------------ + data_path: PathLike + Path where the image is located + + """ + + self.__data_path = data_path + super().__init__() + + @abstractmethod + def as_dask_array(self, chunk_size: Optional[Any] = None) -> da.Array: + """ + Abstract method to return the image as a dask array. + + Parameters + ------------------------ + chunk_size: Optional[Any] + If provided, the image will be rechunked to the desired + chunksize + + Returns + ------------------------ + da.Array + Dask array with the image + + """ + + @abstractmethod + def close_handler(self) -> None: + """ + Abstract method to close the image hander when it's necessary. + + """ + + @abstractmethod + def shape(self) -> Tuple: + """ + Abstract method to return the shape of the image. + + Returns + ------------------------ + Tuple + Tuple with the shape of the image + + """ + + @abstractmethod + def chunks(self) -> Tuple: + """ + Abstract method to return the chunks of the image if it's possible. + + Returns + ------------------------ + Tuple + Tuple with the chunks of the image + + """ + + @property + def data_path(self) -> PathLike: + """ + Getter to return the path where the image is located. + + Returns + ------------------------ + PathLike + Path of the image + + """ + return self.__data_path + + @data_path.setter + def data_path(self, new_data_path: PathLike) -> None: + """ + Setter of the path attribute where the image is located. + + Parameters + ------------------------ + new_data_path: PathLike + New path of the image + + """ + self.__data_path = new_data_path + + +class PngReader(ImageReader): + """ + PngReader class + """ + + def __init__(self, data_path: PathLike) -> None: + """ + Class constructor of image PNG reader. + + Parameters + ------------------------ + data_path: PathLike + Path where the image is located + + """ + super().__init__(data_path) + + def as_dask_array(self, chunk_size: Optional[Any] = None) -> da.Array: + """ + Method to return the image as a dask array. + + Parameters + ------------------------ + chunk_size: Optional[Any] + If provided, the image will be rechunked to the desired + chunksize + + Returns + ------------------------ + da.Array + Dask array with the image + + """ + return daimread(self.data_path, arraytype="numpy") + + @property + def shape(self) -> Tuple: + """ + Abstract method to return the shape of the image. + + Returns + ------------------------ + Tuple + Tuple with the shape of the image + + """ + with pims.open(str(self.data_path)) as imgs: + shape = (len(imgs),) + imgs.frame_shape + + return shape + + @property + def chunks(self) -> Tuple: + """ + Abstract method to return the chunks of the image if it's possible. + + Returns + ------------------------ + Tuple + Tuple with the chunks of the image + + """ + return self.as_dask_array().chunksize + + def close_handler(self) -> None: + """ + Closes image handler + """ + pass + + def __del__(self) -> None: + """Overriding destructor to safely close image""" + self.close_handler() diff --git a/src/aind_smartspim_data_transformation/io/utils.py b/src/aind_smartspim_data_transformation/io/utils.py index a75ba60..968b7f6 100644 --- a/src/aind_smartspim_data_transformation/io/utils.py +++ b/src/aind_smartspim_data_transformation/io/utils.py @@ -4,11 +4,13 @@ import json import os +import platform +import subprocess from typing import Optional import numpy as np -from aind_smartspim_data_transformation._shared.types import ArrayLike +from aind_smartspim_data_transformation.models import ArrayLike, PathLike def add_leading_dim(data: ArrayLike): @@ -106,7 +108,7 @@ def extract_data( return arr[tuple(dynamic_indices)] -def read_json_as_dict(filepath: str) -> dict: +def read_json_as_dict(filepath: PathLike) -> dict: """ Reads a json as dictionary. @@ -131,3 +133,69 @@ def read_json_as_dict(filepath: str) -> dict: dictionary = json.load(json_file) return dictionary + + +def sync_dir_to_s3(directory_to_upload: PathLike, s3_location: str) -> None: + """ + Syncs a local directory to an s3 location by running aws cli in a + subprocess. + + Parameters + ---------- + directory_to_upload : PathLike + s3_location : str + + Returns + ------- + None + + """ + # Upload to s3 + if platform.system() == "Windows": + shell = True + else: + shell = False + + base_command = [ + "aws", + "s3", + "sync", + str(directory_to_upload), + s3_location, + "--only-show-errors", + ] + + subprocess.run(base_command, shell=shell, check=True) + + +def copy_file_to_s3(file_to_upload: PathLike, s3_location: str) -> None: + """ + Syncs a local directory to an s3 location by running aws cli in a + subprocess. + + Parameters + ---------- + file_to_upload : PathLike + s3_location : str + + Returns + ------- + None + + """ + # Upload to s3 + if platform.system() == "Windows": + shell = True + else: + shell = False + + base_command = [ + "aws", + "s3", + "cp", + str(file_to_upload), + s3_location, + "--only-show-errors", + ] + + subprocess.run(base_command, shell=shell, check=True) diff --git a/src/aind_smartspim_data_transformation/models.py b/src/aind_smartspim_data_transformation/models.py index 9f3ed53..d9adbca 100644 --- a/src/aind_smartspim_data_transformation/models.py +++ b/src/aind_smartspim_data_transformation/models.py @@ -1,11 +1,79 @@ """Helpful models used in the ephys compression job""" from enum import Enum +from pathlib import Path +from typing import List, Optional, Union +import numpy as np +from aind_data_transformation.core import BasicJobSettings +from dask import array as da from numcodecs import Blosc +from pydantic import Field + +ArrayLike = Union[da.Array, np.ndarray] +PathLike = Union[str, Path] class CompressorName(str, Enum): """Enum for compression algorithms a user can select""" BLOSC = Blosc.codec_id + + +class SmartspimJobSettings(BasicJobSettings): + """SmartspimCompressionJob settings.""" + + input_source: PathLike = Field( + ..., + description=( + "Source of the SmartSPIM channel data. For example, " + "/scratch/SmartSPIM_695464_2023-10-18_20-30-30/SmartSPIM" + ), + ) + output_directory: PathLike = Field( + ..., + description=("Where to write the data to locally."), + ) + s3_location: Optional[str] = None + num_of_partitions: int = Field( + ..., + description=( + "This script will generate a list of individual stacks, " + "and then partition the list into this number of partitions." + ), + ) + partition_to_process: int = Field( + ..., + description=("Which partition of stacks to process. "), + ) + compressor_name: CompressorName = Field( + default=CompressorName.BLOSC, + description="Type of compressor to use.", + title="Compressor Name.", + ) + # It will be safer if these kwargs fields were objects with known schemas + compressor_kwargs: dict = Field( + default={"cname": "zstd", "clevel": 3, "shuffle": Blosc.SHUFFLE}, + description="Arguments to be used for the compressor.", + title="Compressor Kwargs", + ) + compress_job_save_kwargs: dict = Field( + default={"n_jobs": -1}, # -1 to use all available cpu cores. + description="Arguments for recording save method.", + title="Compress Job Save Kwargs", + ) + chunk_size: List[int] = Field( + default=[128, 128, 128], # Default list with three integers + description="Chunk size in axis, a list of three integers", + title="Chunk Size", + ) + scale_factor: List[int] = Field( + default=[2, 2, 2], # Default list with three integers + description="Scale factors in axis, a list of three integers", + title="Scale Factors", + ) + downsample_levels: int = Field( + default=4, + description="The number of levels of the image pyramid", + title="Downsample Levels", + ) diff --git a/src/aind_smartspim_data_transformation/smartspim_job.py b/src/aind_smartspim_data_transformation/smartspim_job.py index 20ce215..d94430f 100644 --- a/src/aind_smartspim_data_transformation/smartspim_job.py +++ b/src/aind_smartspim_data_transformation/smartspim_job.py @@ -2,227 +2,203 @@ import logging import os +import shutil import sys -from datetime import datetime from pathlib import Path -from typing import Iterator, List, Optional +from time import time +from typing import Any, List, Optional -from aind_data_transformation.core import ( - BasicJobSettings, - GenericEtl, - JobResponse, - get_parser, -) +from aind_data_transformation.core import GenericEtl, JobResponse, get_parser from numcodecs.blosc import Blosc -from pydantic import Field -from aind_smartspim_data_transformation.compress.dask_utils import ( - _cleanup, - get_client, - get_deployment, -) from aind_smartspim_data_transformation.compress.png_to_zarr import ( smartspim_channel_zarr_writer, ) -from aind_smartspim_data_transformation.io import PngReader -from aind_smartspim_data_transformation.models import CompressorName - - -class SmartspimJobSettings(BasicJobSettings): - """SmartspimCompressionJob settings.""" - - # Compress settings - random_seed: Optional[int] = 0 - compressor_name: CompressorName = Field( - default=CompressorName.BLOSC, - description="Type of compressor to use.", - title="Compressor Name.", - ) - # It will be safer if these kwargs fields were objects with known schemas - compressor_kwargs: dict = Field( - default={"cname": "zstd", "clevel": 3, "shuffle": Blosc.SHUFFLE}, - description="Arguments to be used for the compressor.", - title="Compressor Kwargs", - ) - compress_job_save_kwargs: dict = Field( - default={"n_jobs": -1}, # -1 to use all available cpu cores. - description="Arguments for recording save method.", - title="Compress Job Save Kwargs", - ) - chunk_size: int = Field( - default=128, - description="Image chunk size", - title="Image Chunk Size", - ) +from aind_smartspim_data_transformation.io import utils +from aind_smartspim_data_transformation.io.readers import PngReader +from aind_smartspim_data_transformation.models import ( + CompressorName, + SmartspimJobSettings, +) + +logging.basicConfig(level=os.getenv("LOG_LEVEL", "WARNING")) class SmartspimCompressionJob(GenericEtl[SmartspimJobSettings]): - """Main class to handle smartspim data compression""" + """Job to handle compressing and uploading SmartSPIM data.""" - def _get_delayed_channel_stack( - self, channel_paths: List[str], output_dir: str - ) -> Iterator[tuple]: - """ - Reads a stack of PNG images into a delayed zarr dataset. + @staticmethod + def partition_list( + lst: List[Any], num_of_partitions: int + ) -> List[List[Any]]: + """Partitions a list""" + accumulated_list = [] + for _ in range(num_of_partitions): + accumulated_list.append([]) + for list_item_index, list_item in enumerate(lst): + a_index = list_item_index % num_of_partitions + accumulated_list[a_index].append(list_item) + return accumulated_list + + def _get_partitioned_list_of_stack_paths(self) -> List[List[Path]]: + """Scans through the input source and partitions a list of stack + paths that it finds there.""" + all_stack_paths = [] + total_counter = 0 + for channel in [ + p + for p in Path(self.job_settings.input_source) + .joinpath("SmartSPIM") + .iterdir() + if p.is_dir() + ]: + for col in [p for p in channel.iterdir() if p.is_dir()]: + for col_and_row in [p for p in col.iterdir() if p.is_dir()]: + total_counter += 1 + all_stack_paths.append(col_and_row) + # Important to sort paths so every node computes the same list + all_stack_paths.sort(key=lambda x: str(x)) + return self.partition_list( + all_stack_paths, self.job_settings.num_of_partitions + ) - Returns: - Iterator[tuple] - A generator that returns delayed PNG stacks. + @staticmethod + def _get_voxel_resolution(acquisition_path: Path) -> List[float]: + """Get the voxel resolution from an acquisition.json file.""" - """ - for channel_path in channel_paths: - - cols = [ - col_f - for col_f in os.listdir(channel_path) - if Path(channel_path).joinpath(col_f).is_dir() - ] - - for col in cols: - curr_col = channel_path.joinpath(col) - rows_in_cols = [ - row_f - for row_f in os.listdir(curr_col) - if Path(curr_col).joinpath(row_f).is_dir() - ] - for row in rows_in_cols: - curr_row = curr_col.joinpath(row) - delayed_stack = PngReader( - data_path=f"{curr_row}/*.png" - ).as_dask_array() - stack_name = f"{col}_{row.split('_')[-1]}.ome.zarr" - stack_output_path = Path( - f"{output_dir}/{channel_path.stem}" - ) - - yield (delayed_stack, stack_output_path, stack_name) - - def _get_compressor(self) -> Blosc: + if not acquisition_path.is_file(): + raise FileNotFoundError( + f"acquisition.json file not found at: {acquisition_path}" + ) + + acquisition_config = utils.read_json_as_dict(acquisition_path) + + # Grabbing a tile with metadata from acquisition - we assume all + # dataset was acquired with the same resolution + tile_coord_transforms = acquisition_config["tiles"][0][ + "coordinate_transformations" + ] + + scale_transform = [ + x["scale"] for x in tile_coord_transforms if x["type"] == "scale" + ][0] + + x = float(scale_transform[0]) + y = float(scale_transform[1]) + z = float(scale_transform[2]) + + return [z, y, x] + + def _get_compressor(self) -> Optional[Blosc]: """ Utility method to construct a compressor class. Returns ------- - Blosc - An instantiated Blosc compressor. + Blosc | None + An instantiated Blosc compressor. Return None if not set in configs. """ if self.job_settings.compressor_name == CompressorName.BLOSC: return Blosc(**self.job_settings.compressor_kwargs) + else: + return None - return None - - @staticmethod - def _compress_and_write_channels( - read_channel_stacks: Iterator[tuple], - compressor: Blosc, - job_kwargs: dict, - ): + def _write_stacks(self, stacks_to_process: List) -> None: """ - Compresses SmartSPIM image data. - + Write a list of stacks. Parameters ---------- - read_channel_stacks: Iterator[tuple] - Iterator that returns the delayed image stack, - image path and stack name. - """ - - if job_kwargs["n_jobs"] == -1: - job_kwargs["n_jobs"] = os.cpu_count() + stacks_to_process : List - n_workers = job_kwargs["n_jobs"] + Returns + ------- + None - # Instantiating local cluster for parallel writing - deployment = get_deployment() - client, _ = get_client( - deployment, - worker_options=None, # worker_options, - n_workers=n_workers, - processes=True, + """ + compressor = self._get_compressor() + acquisition_path = Path(self.job_settings.input_source).joinpath( + "acquisition.json" ) - print(f"Instantiated client: {client}") - - try: - for delayed_arr, output_path, stack_name in read_channel_stacks: - print( - f"Converting {delayed_arr} from {stack_name} to {output_path}" + voxel_size_zyx = self._get_voxel_resolution( + acquisition_path=acquisition_path + ) + logging.info( + f"Stacks to process: {stacks_to_process}, - Voxel res: " + f"{voxel_size_zyx}" + ) + for stack in stacks_to_process: + logging.info(f"Converting {stack}") + channel_name = stack.parent.parent.name + stack_name = stack.name + + output_path = Path(self.job_settings.output_directory).joinpath( + channel_name + ) + + delayed_stack = PngReader( + data_path=f"{stack}/*.png" + ).as_dask_array() + + smartspim_channel_zarr_writer( + image_data=delayed_stack, + output_path=output_path, + voxel_size=voxel_size_zyx, + final_chunksize=self.job_settings.chunk_size, + scale_factor=self.job_settings.scale_factor, + n_lvls=self.job_settings.downsample_levels, + channel_name=channel_name, + stack_name=f"{stack_name}.ome.zarr", + logger=logging, + writing_options=compressor, + ) + + if self.job_settings.s3_location is not None: + channel_zgroup_file = output_path / ".zgroup" + s3_channel_zgroup_file = ( + f"{self.job_settings.s3_location}/{channel_name}/.zgroup" ) - smartspim_channel_zarr_writer( - image_data=delayed_arr, - output_path=output_path, - voxel_size=[2.0, 1.8, 1.8], - final_chunksize=(128, 128, 128), - scale_factor=[2, 2, 2], - n_lvls=4, - channel_name=output_path.stem, - stack_name=stack_name, - logger=logging, - writing_options=compressor, + logging.info( + f"Uploading {channel_zgroup_file} to " + f"{s3_channel_zgroup_file}" ) - - except Exception as e: - print(f"Error converting array: {e}") - - try: - _cleanup(deployment) - except Exception as e: - print(f"Error shutting down client: {e}") - - def _compress_raw_data(self) -> None: - """Compresses smartspim data""" - - # Clip the data - logging.info("Converting PNG to OMEZarr. This may take some minutes.") - output_compressed_data = self.job_settings.output_directory - - raw_path = self.job_settings.input_source / "SmartSPIM" - print(f"Raw path: {raw_path} " f"OS: {raw_path}") - - channel_paths = [ - Path(raw_path).joinpath(folder) - for folder in os.listdir(raw_path) - if Path(raw_path).joinpath(folder).is_dir() + utils.copy_file_to_s3( + channel_zgroup_file, s3_channel_zgroup_file + ) + ome_zarr_stack_name = f"{stack_name}.ome.zarr" + ome_zarr_stack_path = output_path.joinpath(ome_zarr_stack_name) + s3_stack_dir = ( + f"{self.job_settings.s3_location}/{channel_name}/" + f"{ome_zarr_stack_name}" + ) + logging.info( + f"Uploading {ome_zarr_stack_path} to {s3_stack_dir}" + ) + utils.sync_dir_to_s3(ome_zarr_stack_path, s3_stack_dir) + logging.info(f"Removing: {ome_zarr_stack_path}") + # Remove stack if uploaded to s3. We can potentially do all + # the stacks in the partition in parallel using dask to speed + # this up + shutil.rmtree(ome_zarr_stack_path) + + def run_job(self): + """Main entrypoint to run the job.""" + job_start_time = time() + + partitioned_list = self._get_partitioned_list_of_stack_paths() + stacks_to_process = partitioned_list[ + self.job_settings.partition_to_process ] - # Get channel stack iterators and delayed arrays - read_delayed_channel_stacks = self._get_delayed_channel_stack( - channel_paths=channel_paths, - output_dir=output_compressed_data, - ) - - # Getting compressors - compressor = self._get_compressor() - - # Writing compressed stacks - self._compress_and_write_channels( - read_channel_stacks=read_delayed_channel_stacks, - compressor=compressor, - job_kwargs=self.job_settings.compress_job_save_kwargs, - ) - logging.info("Finished compressing source data.") - - def run_job(self) -> JobResponse: - """ - Main public method to run the compression job - Returns - ------- - JobResponse - Information about the job that can be used for metadata downstream. - - """ - job_start_time = datetime.now() - self._compress_raw_data() - job_end_time = datetime.now() + self._write_stacks(stacks_to_process=stacks_to_process) + total_job_duration = time() - job_start_time return JobResponse( - status_code=200, - message=f"Job finished in: {job_end_time-job_start_time}", - data=None, + status_code=200, message=f"Job finished in {total_job_duration}" ) -def main(): +# TODO: Add this to core aind_data_transformation class +def job_entrypoint(sys_args: list): """Main function""" - sys_args = sys.argv[1:] parser = get_parser() cli_args = parser.parse_args(sys_args) if cli_args.job_settings is not None: @@ -242,4 +218,4 @@ def main(): if __name__ == "__main__": - main() + job_entrypoint(sys.argv[1:]) diff --git a/tests/compress/test_dask_utils.py b/tests/compress/test_dask_utils.py deleted file mode 100644 index 772a6ac..0000000 --- a/tests/compress/test_dask_utils.py +++ /dev/null @@ -1,322 +0,0 @@ -"""Tests dask utils""" - -import unittest -from unittest.mock import MagicMock, patch - -from distributed import Client - -from aind_smartspim_data_transformation.compress import dask_utils - - -class DaskUtilsTest(unittest.TestCase): - """Class for testing the zarr writer""" - - def test_get_local_deployment(self): - """Tests getting a deployment""" - deployment = dask_utils.get_deployment() - - self.assertEqual(dask_utils.Deployment.LOCAL.value, deployment) - - @patch.dict("os.environ", {"SLURM_JOBID": "000"}) - def test_get_allen_deploymet(self): - """Tests getting a deployment on the Allen HPC""" - deployment = dask_utils.get_deployment() - - self.assertEqual(dask_utils.Deployment.SLURM.value, deployment) - - @patch("aind_smartspim_data_transformation.compress.dask_utils.get_client") - def test_get_local_client(self, mock_client: MagicMock): - """Tests getting a local client""" - mock_client.return_value = (Client, 0) - - deployment = dask_utils.get_deployment() - client, _ = dask_utils.get_client( - deployment=deployment, - worker_options=0, - n_workers=1, - processes=True, - ) - - self.assertEqual(client, Client) - - def test_get_client_fail(self): - """Tests fail getting a local client""" - - with self.assertRaises(NotImplementedError): - dask_utils.get_client( - deployment="UnknownDeployment", - worker_options=0, - n_workers=1, - processes=True, - ) - - @patch.dict("os.environ", {"SLURM_JOBID": "000"}) - @patch("distributed.Client") - def test_get_slurm_client_mpi_failure(self, mock_client: MagicMock): - """Tests getting a slurm client""" - mock_client.return_value = (Client, 0) - - deployment = dask_utils.get_deployment() - - with self.assertRaises(ImportError): - dask_utils.get_client( - deployment=deployment, - worker_options=0, - n_workers=1, - processes=True, - ) - - @patch.dict("os.environ", {"SLURM_JOBID": "000"}) - @patch( - "aind_smartspim_data_transformation.compress.dask_utils.DASK_MPI_INSTALLED", - new=True, - ) - @patch.dict("sys.modules", {"dask_mpi": None}) - @patch("aind_smartspim_data_transformation.compress.dask_utils.get_client") - @patch("distributed.Client") - def test_get_slurm_client_mpi( - self, mock_client: MagicMock, mock_get_client: MagicMock - ): - """Tests getting a slurm client""" - mock_client.return_value = (Client, 0) - mock_get_client.return_value = Client - - deployment = dask_utils.get_deployment() - slurm_client = dask_utils.get_client( - deployment=deployment, - worker_options=None, - n_workers=1, - processes=True, - ) - - self.assertEqual(slurm_client, Client) - mock_get_client.assert_called_once_with( - deployment=deployment, - worker_options=None, - n_workers=1, - processes=True, - ) - - @patch("requests.delete") - def test_cancel_slurm_job_success(self, mock_requests_delete: MagicMock): - """ - Tests cancelling a slurm job successfully - """ - mock_response = MagicMock() - mock_response.status_code = 200 - mock_requests_delete.return_value = mock_response - - job_id = "123" - api_url = "http://myhost:80/api/slurm/v0.0.36" - headers = {"Authorization": "Bearer token"} - - response = dask_utils.cancel_slurm_job(job_id, api_url, headers) - - self.assertEqual(response.status_code, mock_response.status_code) - mock_requests_delete.assert_called_once_with( - f"{api_url}/job/{job_id}", headers=headers - ) - - @patch("requests.delete") - def test_cancel_slurm_job_failure(self, mock_requests_delete: MagicMock): - """ - Tests cancelling slurm job with - mock job failure - """ - mock_response = MagicMock() - mock_response.status_code = 500 - mock_requests_delete.return_value = mock_response - - job_id = "123" - api_url = "http://myhost:80/api/slurm/v0.0.36" - headers = {"Authorization": "Bearer token"} - - response = dask_utils.cancel_slurm_job(job_id, api_url, headers) - self.assertEqual(response.status_code, mock_response.status_code) - - mock_requests_delete.assert_called_once_with( - f"{api_url}/job/{job_id}", headers=headers - ) - - @patch.dict( - "os.environ", - { - "SLURM_JOBID": "000", - "HPC_HOST": "example.com", - "HPC_PORT": "80", - "HPC_API_ENDPOINT": "api", - "HPC_USERNAME": "username", - "HPC_PASSWORD": "password", - "HPC_TOKEN": "token", - }, - ) - @patch("os.getenv") - @patch( - "aind_smartspim_data_transformation.compress.dask_utils.cancel_slurm_job" - ) - def test_cleanup_slurm_with_env_vars( - self, mock_cancel_slurm_job: MagicMock, mock_getenv: MagicMock - ): - """ - Cleaning up slurm job with - environment variables - """ - mock_getenv.side_effect = lambda x: { - "SLURM_JOBID": "123", - "HPC_HOST": "example.com", - "HPC_PORT": "80", - "HPC_API_ENDPOINT": "api", - "HPC_USERNAME": "username", - "HPC_PASSWORD": "password", - "HPC_TOKEN": "token", - }.get(x) - - # Set up mock response for cancel_slurm_job - mock_response = MagicMock() - mock_response.status_code = 200 - mock_cancel_slurm_job.return_value = mock_response - - dask_utils._cleanup(deployment=dask_utils.Deployment.SLURM.value) - - mock_cancel_slurm_job.assert_called_once_with( - "123", - "http://example.com:80/api", - { - "X-SLURM-USER-NAME": "username", - "X-SLURM-USER-PASSWORD": "password", - "X-SLURM-USER-TOKEN": "token", - }, - ) - - @patch.dict( - "os.environ", - { - "SLURM_JOBID": "000", - "HPC_HOST": "example.com", - "HPC_PORT": "80", - "HPC_API_ENDPOINT": "api", - "HPC_USERNAME": "username", - "HPC_PASSWORD": "password", - "HPC_TOKEN": "token", - }, - ) - @patch("os.getenv") - @patch( - "aind_smartspim_data_transformation.compress.dask_utils.cancel_slurm_job" - ) - @patch("aind_smartspim_data_transformation.compress.dask_utils.logging") - def test_cleanup_slurm_with_env_vars_failed( - self, - mock_logging: MagicMock, - mock_cancel_slurm_job: MagicMock, - mock_getenv: MagicMock, - ): - """ - Tests failure cleaning up slurm job with - environment variables - """ - mock_getenv.side_effect = lambda x: { - "SLURM_JOBID": "123", - "HPC_HOST": "example.com", - "HPC_PORT": "80", - "HPC_API_ENDPOINT": "api", - "HPC_USERNAME": "username", - "HPC_PASSWORD": "password", - "HPC_TOKEN": "token", - }.get(x) - - # Set up mock response for cancel_slurm_job - mock_response = MagicMock() - mock_response.status_code = 404 - mock_response.text = "test" - mock_cancel_slurm_job.return_value = mock_response - - dask_utils._cleanup(deployment=dask_utils.Deployment.SLURM.value) - - mock_cancel_slurm_job.assert_called_once_with( - "123", - "http://example.com:80/api", - { - "X-SLURM-USER-NAME": "username", - "X-SLURM-USER-PASSWORD": "password", - "X-SLURM-USER-TOKEN": "token", - }, - ) - mock_logging.error.assert_called_once_with( - "Failed to cancel SLURM job 123: test" - ) - - @patch.dict("os.environ", {"SLURM_JOBID": "000"}) - @patch("aind_smartspim_data_transformation.compress.dask_utils.logging") - @patch("os.getenv") - def test_cleanup_slurm_without_env_vars( - self, mock_getenv: MagicMock, mock_logging: MagicMock - ): - """ - Tests cleaning up slurm without - environment variables - """ - mock_getenv.side_effect = lambda x: { - "SLURM_JOBID": "123", - "HPC_HOST": "example.com", - "HPC_PORT": "80", - "HPC_API_ENDPOINT": "api", - "HPC_USERNAME": "username", - "HPC_PASSWORD": "password", - "HPC_TOKEN": "token", - }.get(x) - - dask_utils._cleanup(deployment=dask_utils.Deployment.SLURM.value) - mock_logging.error.assert_called_once_with( - "Failed to get SLURM env vars to cleanup: 'HPC_HOST'" - ) - - @patch("os.getenv") - @patch("aind_smartspim_data_transformation.compress.dask_utils.logging") - def test_cleanup_local( - self, mock_logging: MagicMock, mock_getenv: MagicMock - ): - """ - Tests cleaning up a local cluster - """ - mock_getenv.return_value = None - - dask_utils._cleanup(deployment=dask_utils.Deployment.LOCAL.value) - - mock_logging.info.assert_not_called() - - @patch("os.getenv") - @patch( - "aind_smartspim_data_transformation.compress.dask_utils.LOGGER.info" - ) - @patch("distributed.Client") - def test_log_dashboard_address( - self, - mock_Client: MagicMock, - mock_logger_info: MagicMock, - mock_getenv: MagicMock, - ): - """ - Tests log dashboard address - """ - mock_getenv.return_value = "testuser" - - mock_client = MagicMock() - mock_Client.return_value = mock_client - - mock_client.scheduler_info.return_value = { - "services": {"dashboard": 8787} - } - - mock_client.run_on_scheduler.return_value = "scheduler-host" - - dask_utils.log_dashboard_address(client=mock_client) - - mock_logger_info.assert_called_once_with( - "To access the dashboard, run the following in " - "a terminal: ssh -L 8787:scheduler-host:8787 testuser@hpc-login " - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/resources/SmartSPIM_000000_2024-06-05_07-56-54/acquisition.json b/tests/resources/SmartSPIM_000000_2024-06-05_07-56-54/acquisition.json new file mode 100644 index 0000000..afc79a8 --- /dev/null +++ b/tests/resources/SmartSPIM_000000_2024-06-05_07-56-54/acquisition.json @@ -0,0 +1,2352 @@ +{ + "active_objectives": null, + "axes": [ + { + "dimension": 2, + "direction": "Left_to_right", + "name": "X", + "unit": "micrometer" + }, + { + "dimension": 1, + "direction": "Posterior_to_anterior", + "name": "Y", + "unit": "micrometer" + }, + { + "dimension": 0, + "direction": "Superior_to_inferior", + "name": "Z", + "unit": "micrometer" + } + ], + "chamber_immersion": { + "medium": "Cargille 1.52", + "refractive_index": 1.522 + }, + "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/imaging/acquisition.py", + "experimenter_full_name": [ + "Erica Peterson" + ], + "external_storage_directory": "", + "instrument_id": "SmartSPIM3-2", + "local_storage_directory": "D:/SmartSPIM_Data", + "notes": null, + "processing_steps": null, + "sample_immersion": { + "medium": "EasyIndex", + "refractive_index": 1.5199 + }, + "schema_version": "0.4.6", + "session_end_time": "2023-10-19T12:00:55", + "session_start_time": "2023-10-18T20:30:30", + "session_type": null, + "specimen_id": "", + "subject_id": "695464", + "tiles": [ + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/471320.0/471320.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/471320.0/471320.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/568520.0/568520.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/568520.0/568520.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/471320.0/471320.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/471320.0/471320.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/471320.0/471320.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/503720.0/503720.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/503720.0/503720.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/503720.0/503720.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/536120.0/536120.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/536120.0/536120.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/471320.0/471320.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/536120.0/536120.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/568520.0/568520.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/568520.0/568520.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 72741.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/568520.0/568520.0_727410.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/471320.0/471320.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/471320.0/471320.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/471320.0/471320.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/503720.0/503720.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/503720.0/503720.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/503720.0/503720.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/503720.0/503720.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/536120.0/536120.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/536120.0/536120.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/536120.0/536120.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/568520.0/568520.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/568520.0/568520.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 75333.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/568520.0/568520.0_753330.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/471320.0/471320.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/471320.0/471320.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/471320.0/471320.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/503720.0/503720.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/503720.0/503720.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/503720.0/503720.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/503720.0/503720.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/536120.0/536120.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/536120.0/536120.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/536120.0/536120.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/568520.0/568520.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/568520.0/568520.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 77925.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/568520.0/568520.0_779250.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/471320.0/471320.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/471320.0/471320.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/503720.0/503720.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/471320.0/471320.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/503720.0/503720.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/503720.0/503720.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/503720.0/503720.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/536120.0/536120.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/536120.0/536120.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/536120.0/536120.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/568520.0/568520.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/568520.0/568520.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 80517.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/568520.0/568520.0_805170.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/536120.0/536120.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/471320.0/471320.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/471320.0/471320.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 47132.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/471320.0/471320.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/503720.0/503720.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/503720.0/503720.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 50372.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/503720.0/503720.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/536120.0/536120.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/536120.0/536120.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/536120.0/536120.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/568520.0/568520.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/536120.0/536120.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "561", + "filter_wheel_index": 1, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 561, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_561_Em_600/568520.0/568520.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 83109.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/568520.0/568520.0_831090.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "639", + "filter_wheel_index": 2, + "laser_power": 100.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 639, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 53612.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_639_Em_680/536120.0/536120.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + }, + { + "channel": { + "channel_name": "488", + "filter_wheel_index": 0, + "laser_power": 75.0, + "laser_power_unit": "milliwatt", + "laser_wavelength": 488, + "laser_wavelength_unit": "nanometer" + }, + "coordinate_transformations": [ + { + "translation": [ + 56852.0, + 70149.0, + 170.7 + ], + "type": "translation" + }, + { + "scale": [ + 1.8, + 1.8, + 2.0 + ], + "type": "scale" + } + ], + "file_name": "Ex_488_Em_525/568520.0/568520.0_701490.0/", + "imaging_angle": 0, + "imaging_angle_unit": "degree", + "notes": "\nLaser power is in percentage of total, it needs calibration" + } + ] +} \ No newline at end of file diff --git a/tests/resources/test_configs.json b/tests/resources/test_configs.json new file mode 100644 index 0000000..380d598 --- /dev/null +++ b/tests/resources/test_configs.json @@ -0,0 +1,6 @@ +{ + "input_source": "tests/resources/SmartSPIM_000000_2024-06-05_07-56-54", + "output_directory": "fake_output_dir", + "num_of_partitions": 4, + "partition_to_process": 0 +} \ No newline at end of file diff --git a/tests/compress/__init__.py b/tests/test_compress/__init__.py similarity index 100% rename from tests/compress/__init__.py rename to tests/test_compress/__init__.py diff --git a/tests/compress/test_zarr_writer.py b/tests/test_compress/test_zarr_writer.py similarity index 97% rename from tests/compress/test_zarr_writer.py rename to tests/test_compress/test_zarr_writer.py index c0c768d..9b55d1e 100644 --- a/tests/compress/test_zarr_writer.py +++ b/tests/test_compress/test_zarr_writer.py @@ -59,3 +59,7 @@ def test_closer_to_target(self): self.assertEqual(test_arr_1.shape, shape_close_target) self.assertEqual(test_arr_1.shape, shape_close_target_2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..0e5f940 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,50 @@ +"""Tests for the SmartSPIM data transfer""" + +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +from aind_smartspim_data_transformation.smartspim_job import ( + SmartspimCompressionJob, + SmartspimJobSettings, +) + +TEST_DIR = Path(os.path.dirname(os.path.realpath(__file__))) / "resources" +DATA_DIR = TEST_DIR / "SmartSPIM_000000_2024-06-05_07-56-54" + + +class SmartspimCompressionTest(unittest.TestCase): + """Class for testing the data transform""" + + @classmethod + def setUpClass(cls) -> None: + """Setup basic job settings and job that can be used across tests""" + # Folder to test the zarr writing from PNGs + cls.temp_folder = tempfile.mkdtemp(prefix="unittest_") + + basic_job_settings = SmartspimJobSettings( + input_source=DATA_DIR, + output_directory=Path(cls.temp_folder), + num_of_partitions=4, + partition_to_process=0, + ) + cls.basic_job_settings = basic_job_settings + cls.basic_job = SmartspimCompressionJob( + job_settings=basic_job_settings + ) + + def test_run_job(self): + """Tests SmartSPIM compression and zarr writing""" + self.basic_job.run_job() + + @classmethod + def tearDownClass(cls) -> None: + """Tear down class method to clean up""" + if os.path.exists(cls.temp_folder): + shutil.rmtree(cls.temp_folder, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/io/__init__.py b/tests/test_io/__init__.py similarity index 100% rename from tests/io/__init__.py rename to tests/test_io/__init__.py diff --git a/tests/test_io/test_readers.py b/tests/test_io/test_readers.py new file mode 100644 index 0000000..304e50a --- /dev/null +++ b/tests/test_io/test_readers.py @@ -0,0 +1,54 @@ +"""Tests methods in io.readers module""" + +import os +import unittest +from pathlib import Path + +from aind_smartspim_data_transformation.io.readers import PngReader + +RESOURCES_DIR = ( + Path(os.path.dirname(os.path.realpath(__file__))) / ".." / "resources" +) + +STACK_DIR = ( + RESOURCES_DIR + / "SmartSPIM_000000_2024-06-05_07-56-54" + / "SmartSPIM" + / "Ex_445_Em_469" + / "432380" + / "432380_504340" + / "*.png" +) + + +class TestPngReader(unittest.TestCase): + """Tests methods in PngReader class""" + + @classmethod + def setUpClass(cls): + """Sets up class with common reader""" + cls.reader = PngReader(STACK_DIR) + + def test_shape(self): + """Tests shape method""" + expected_shape = (2, 1600, 2000) + self.assertEqual(expected_shape, self.reader.shape) + + def test_chunks(self): + """Tests chunks method""" + expected_chunks = (1, 1600, 2000) + self.assertEqual(expected_chunks, self.reader.chunks) + + def test_data_path(self): + """Tests data_path method""" + self.assertEqual(STACK_DIR, self.reader.data_path) + + def test_data_path_setter(self): + """Tests data_path_setter""" + reader = PngReader(STACK_DIR) + reader.data_path = "tests" + self.assertEqual("tests", reader.data_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/io/test_utils.py b/tests/test_io/test_utils.py similarity index 87% rename from tests/io/test_utils.py rename to tests/test_io/test_utils.py index c715e1f..9653a46 100644 --- a/tests/io/test_utils.py +++ b/tests/test_io/test_utils.py @@ -11,17 +11,16 @@ from aind_smartspim_data_transformation.io import utils +RESOURCES_DIR = ( + Path(os.path.dirname(os.path.realpath(__file__))) / ".." / "resources" +) + +JSON_FILE_PATH = RESOURCES_DIR / "local_json.json" + class IoUtilitiesTest(unittest.TestCase): """Class for testing the io utilities""" - def setUp(self): - """Setting up temporary folder directory""" - current_path = Path(os.path.abspath(__file__)).parent - self.test_local_json_path = current_path.joinpath( - "../resources/local_json.json" - ) - def test_add_leading_dim(self): """ Tests that a new dimension is added @@ -80,5 +79,9 @@ def test_read_json_as_dict(self): Tests successful reading of a dictionary """ expected_result = {"some_key": "some_value"} - result = utils.read_json_as_dict(self.test_local_json_path) + result = utils.read_json_as_dict(JSON_FILE_PATH) self.assertEqual(expected_result, result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_smartspim_job.py b/tests/test_smartspim_job.py index 5c60f46..bdcefbf 100644 --- a/tests/test_smartspim_job.py +++ b/tests/test_smartspim_job.py @@ -1,21 +1,22 @@ """Tests for the SmartSPIM data transfer""" +import json import os -import shutil -import tempfile import unittest from pathlib import Path +from unittest.mock import MagicMock, patch -import numpy as np from numcodecs.blosc import Blosc +from aind_smartspim_data_transformation.models import SmartspimJobSettings from aind_smartspim_data_transformation.smartspim_job import ( SmartspimCompressionJob, - SmartspimJobSettings, + job_entrypoint, ) -TEST_DIR = Path(os.path.dirname(os.path.realpath(__file__))) / "resources" -DATA_DIR = TEST_DIR / "SmartSPIM_000000_2024-06-05_07-56-54" +RESOURCES_DIR = Path(os.path.dirname(os.path.realpath(__file__))) / "resources" + +DATA_DIR = RESOURCES_DIR / "SmartSPIM_000000_2024-06-05_07-56-54" class SmartspimCompressionTest(unittest.TestCase): @@ -24,73 +25,180 @@ class SmartspimCompressionTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: """Setup basic job settings and job that can be used across tests""" - # Folder to test the zarr writing from PNGs - cls.temp_folder = tempfile.mkdtemp(prefix="unittest_") basic_job_settings = SmartspimJobSettings( input_source=DATA_DIR, - output_directory=Path(cls.temp_folder), + output_directory="fake_output_dir", + num_of_partitions=4, + partition_to_process=0, ) cls.basic_job_settings = basic_job_settings cls.basic_job = SmartspimCompressionJob( job_settings=basic_job_settings ) - def test_get_delayed_channel_stack(self): - """Tests reading stacks of png images.""" - - raw_path = self.basic_job.job_settings.input_source / "SmartSPIM" - - channel_paths = [ - Path(raw_path).joinpath(folder) for folder in os.listdir(raw_path) + def test_partition_list(self): + """Tests partition list method""" + test_list = [f"ID: {x}" for x in range(75)] + output_list1 = self.basic_job.partition_list( + test_list, num_of_partitions=5 + ) + output_list2 = self.basic_job.partition_list( + test_list, num_of_partitions=2 + ) + flat_output1 = [x for xs in output_list1 for x in xs] + flat_output2 = [x for xs in output_list2 for x in xs] + self.assertEqual(5, len(output_list1)) + self.assertEqual(2, len(output_list2)) + self.assertCountEqual(test_list, flat_output1) + self.assertCountEqual(test_list, flat_output2) + + def test_get_partitioned_list_of_stack_paths(self): + """Tests _get_partitioned_list_of_stack_paths""" + stack_paths = self.basic_job._get_partitioned_list_of_stack_paths() + flat_list_of_paths = [x for xs in stack_paths for x in xs] + channel_dir1 = DATA_DIR / "SmartSPIM" / "Ex_445_Em_469" + channel_dir2 = DATA_DIR / "SmartSPIM" / "Ex_561_Em_600" + expected_flat_list = [ + channel_dir1 / "432380" / "432380_504340", + channel_dir2 / "432380" / "432380_504340", + channel_dir1 / "432380" / "432380_530260", + channel_dir2 / "432380" / "432380_530260", + channel_dir1 / "464780" / "464780_504340", + channel_dir2 / "464780" / "464780_504340", + channel_dir1 / "464780" / "464780_530260", + channel_dir2 / "464780" / "464780_530260", ] + self.assertEqual(4, len(stack_paths)) + self.assertEqual(expected_flat_list, flat_list_of_paths) - # Get channel stack iterators and delayed arrays - read_delayed_channel_stacks = ( - self.basic_job._get_delayed_channel_stack( - channel_paths=channel_paths, - output_dir=self.basic_job.job_settings.output_directory, - ) - ) + def test_get_voxel_resolution(self): + """Tests _get_voxel_resolution""" + acq_path = DATA_DIR / "acquisition.json" + voxel_res = self.basic_job._get_voxel_resolution(acq_path) + expected_res = [2.0, 1.8, 1.8] + self.assertEqual(expected_res, voxel_res) - stacked_shape = (2, 1600, 2000) - dtype = np.uint16 - extensions = [".ome", ".zarr"] + def test_get_voxel_resolution_error(self): + """Tests _get_voxel_resolution when file not found""" + acq_path = DATA_DIR / "acq.json" # No acquisition.json file here + with self.assertRaises(FileNotFoundError): + self.basic_job._get_voxel_resolution(acq_path) - for delayed_arr, _, stack_name in read_delayed_channel_stacks: - self.assertEqual(stacked_shape, delayed_arr.shape) - self.assertEqual(dtype, delayed_arr.dtype) - self.assertEqual(extensions, Path(stack_name).suffixes) + def test_get_compressor(self): + """Tests _get_compressor method""" - def test_compressor(self): - """Test compression with Blosc""" compressor = self.basic_job._get_compressor() - current_blosc = Blosc(**self.basic_job.job_settings.compressor_kwargs) - self.assertEqual(compressor, current_blosc) - - def test_getting_compressor_fail(self): - """Test failed compression with Blosc""" - - with self.assertRaises(Exception): - # Failed blosc compressor - failed_basic_job_settings = SmartspimJobSettings( - input_source=DATA_DIR, - output_directory=Path("output_dir"), - compressor_name="not_blosc", - ) - - failed_basic_job_settings = failed_basic_job_settings - SmartspimCompressionJob(job_settings=failed_basic_job_settings) + expected_compressor = Blosc( + cname="zstd", clevel=3, shuffle=Blosc.SHUFFLE, blocksize=0 + ) + self.assertEqual(expected_compressor, compressor) - def test_run_job(self): - """Tests SmartSPIM compression and zarr writing""" - self.basic_job.run_job() + def test_get_compressor_none(self): + """Tests _get_compressor method returns None if no config set""" - @classmethod - def tearDownClass(cls) -> None: - """Tear down class method to clean up""" - if os.path.exists(cls.temp_folder): - shutil.rmtree(cls.temp_folder, ignore_errors=True) + job_settings = SmartspimJobSettings.model_construct( + compressor_name="foo" + ) + job = SmartspimCompressionJob(job_settings=job_settings) + compressor = job._get_compressor() + self.assertIsNone(compressor) + + @patch( + "aind_smartspim_data_transformation.smartspim_job" + ".smartspim_channel_zarr_writer" + ) + def test_run_job(self, mock_zarr_write: MagicMock): + """Test run_job method""" + response = self.basic_job.run_job() + mock_zarr_write.assert_called() + self.assertIsNotNone(response) + + @patch( + "aind_smartspim_data_transformation.smartspim_job" + ".SmartspimCompressionJob.run_job" + ) + @patch("aind_smartspim_data_transformation.smartspim_job.get_parser") + def test_job_entrypoint_job_settings( + self, mock_get_parser: MagicMock, mock_run_job: MagicMock + ): + """Tests job_entrypoint with json settings""" + + json_settings = json.dumps( + { + "input_source": "input", + "output_directory": "output_dir", + "num_of_partitions": 4, + "partition_to_process": 1, + } + ) + mock_parser = MagicMock() + mock_parser.parse_args.return_value.job_settings = json_settings + mock_get_parser.return_value = mock_parser + sys_args = ["", "-j", f"' {json_settings}' "] + mock_response = MagicMock() + mock_response.model_dump_json.return_value = json.dumps( + {"message": "ran job"} + ) + mock_run_job.return_value = mock_response + job_entrypoint(sys_args) + mock_run_job.assert_called() + + @patch( + "aind_smartspim_data_transformation.smartspim_job" + ".SmartspimCompressionJob.run_job" + ) + @patch("aind_smartspim_data_transformation.smartspim_job.get_parser") + def test_job_entrypoint_config_file( + self, mock_get_parser: MagicMock, mock_run_job: MagicMock + ): + """Tests job_entrypoint with config file""" + config_file = RESOURCES_DIR / "test_configs.json" + mock_parser = MagicMock() + mock_parser.parse_args.return_value.job_settings = None + mock_parser.parse_args.return_value.config_file = config_file + mock_get_parser.return_value = mock_parser + sys_args = ["", "--config-file", f"{config_file}"] + mock_response = MagicMock() + mock_response.model_dump_json.return_value = json.dumps( + {"message": "ran job"} + ) + mock_run_job.return_value = mock_response + job_entrypoint(sys_args) + mock_run_job.assert_called() + + @patch.dict( + os.environ, + { + "TRANSFORMATION_JOB_INPUT_SOURCE": "input", + "TRANSFORMATION_JOB_OUTPUT_DIRECTORY": "output_dir", + "TRANSFORMATION_JOB_NUM_OF_PARTITIONS": "4", + "TRANSFORMATION_JOB_PARTITION_TO_PROCESS": "1", + }, + clear=True, + ) + @patch( + "aind_smartspim_data_transformation.smartspim_job" + ".SmartspimCompressionJob.run_job" + ) + @patch("aind_smartspim_data_transformation.smartspim_job.get_parser") + def test_job_entrypoint_env_vars( + self, mock_get_parser: MagicMock, mock_run_job: MagicMock + ): + """Tests job_entrypoint method""" + + mock_parser = MagicMock() + mock_parser.parse_args.return_value.job_settings = None + mock_parser.parse_args.return_value.config_file = None + mock_get_parser.return_value = mock_parser + sys_args = [""] + mock_response = MagicMock() + mock_response.model_dump_json.return_value = json.dumps( + {"message": "ran job"} + ) + mock_run_job.return_value = mock_response + job_entrypoint(sys_args) + mock_run_job.assert_called() if __name__ == "__main__":