diff --git a/.gitignore b/.gitignore index 0686d4d..48a0c44 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ __pycache__ # Data data *.csv +.zarrcade # Zarrcade databases *.db diff --git a/README.md b/README.md index 9b97471..161fe45 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,12 @@ Zarrcade is a web application for easily browsing, searching, and visualizing co ## Getting Started + ### 1. Install miniforge [Install miniforge](https://docs.conda.io/en/latest/miniforge.html) if you don't already have it. + ### 2. Initialize the conda environment ```bash @@ -30,9 +32,15 @@ conda env create -f environment.yml conda activate zarrcade ``` + +### (Optional) Try an example + +See the [Example](#example) section below to try out the example before working with your own data. + + ### 3. Create OME-Zarr images -If necessary, convert your image(s) to OME-Zarr format, e.g. using bioformats2raw: +If your images are not already in OME-Zarr format, you will need to convert them, e.g. using bioformats2raw: ```bash bioformats2raw -w 128 -h 128 -z 64 --compression zlib /path/to/input /path/to/zarr @@ -40,6 +48,7 @@ bioformats2raw -w 128 -h 128 -z 64 --compression zlib /path/to/input /path/to/za If you have many images to convert, we recommend using the [nf-omezarr Nextflow pipeline](https://github.com/JaneliaSciComp/nf-omezarr) to efficiently run bioformats2raw on a collection of images. This pipeline also lets you scale the conversion processes to your available compute resources (cluster, cloud, etc). + ### 4. Import images and metadata into Zarrcade You can import images into Zarrcade using the provided command line script: @@ -48,9 +57,7 @@ You can import images into Zarrcade using the provided command line script: bin/import.py -d /root/data/dir -c mycollection ``` -This will automatically create a local Sqlite database containing a Zarrcade **collection** named "mycollection" and populate it with information about the images in the specified directory. - -By default, this will also create MIPs and thumbnails for each image in a folder named `.zarrcade` within the root data directory. You can change this location by setting the `--aux-path` parameter. You can disable the creation of MIPs and thumbnails by setting the `--no-aux` flag. The brightness of the MIPs can be adjusted using the `--p-lower` and `--p-upper` parameters. +This will automatically create a local Sqlite database containing a Zarrcade **collection** named "mycollection" and populate it with information about the images in the specified directory. By default, this will also create MIPs and thumbnails for each image in `./static/.zarrcade`. To add extra metadata about the images, you can provide a CSV file with the `-i` flag: @@ -66,11 +73,8 @@ relative/path/to/ome1.zarr,JKF6363,Blu relative/path/to/ome2.zarr,JDH3562,Blu ``` -To try an example, use the following command: +Read more about the import options in the [Data Import](./docs/DataImport.md) section of the documentation. -```bash -bin/import.py -d s3://janelia-data-examples/fly-efish -c flyefish -m docs/flyefish-example.csv -``` ### 5. Run the Zarrcade web application @@ -83,6 +87,24 @@ uvicorn zarrcade.serve:app --host 0.0.0.0 --reload Your images and annotations will be indexed and browseable at [http://0.0.0.0:8000](http://0.0.0.0:8000). Read the documentation below for more details on how to configure the web UI and deploy the service in production. +## Example + +To try an example, follow steps 1 and 2 above and use the following command to import the example data: + +```bash +bin/import.py -d s3://janelia-data-examples/fly-efish -c flyefish -m docs/flyefish-example.csv +``` + +Copy the example settings.yaml file to your working directory and start the server: + +```bash +cp docs/settings.yaml.example settings.yaml +uvicorn zarrcade.serve:app --host 0.0.0.0 --reload +``` + +The example should be visible at [http://0.0.0.0:8000](http://0.0.0.0:8000). + + ## Documentation * [Overview](./docs/Overview.md) - learn about the data model and overall architecture diff --git a/bin/import.py b/bin/import.py index a4568c2..dd2595c 100755 --- a/bin/import.py +++ b/bin/import.py @@ -61,8 +61,8 @@ def slugify(value, allow_unicode=False): help='Path to the CSV file containing additional metadata') parser.add_argument('-x', '--no-aux', action=argparse.BooleanOptionalAction, default=False, help="Don't create auxiliary images or thumbnails.") - parser.add_argument('-a', '--aux-path', type=str, default=".zarrcade", - help='Path to the folder containing auxiliary images (non-absolute paths are relative to data_url).') + parser.add_argument('-a', '--aux-path', type=str, default="static/.zarrcade", + help='Local path to the folder for auxiliary images.') parser.add_argument('--aux-image-name', type=str, default='zmax.png', help='Filename of the main auxiliary image.') parser.add_argument('--thumbnail-name', type=str, default='zmax_sm.jpg', @@ -161,35 +161,33 @@ def get_aux_path(zarr_path, filename): logger.info(f"Processing {image.path}") metadata = image.image_metadata updated_obj = {} - aux_abspath = None + aux_path = None if args.aux_image_name and (not metadata or not metadata.aux_image_path): aux_path = get_aux_path(image.path, args.aux_image_name) - aux_abspath = fs.get_absolute_path(aux_path) updated_obj['aux_image_path'] = aux_path if fs.exists(aux_path): - logger.trace(f"Found auxiliary file: {aux_abspath}") + logger.trace(f"Found auxiliary file: {aux_path}") else: - logger.trace(f"Creating auxiliary file: {aux_abspath}") - zarr_path = fs.get_absolute_path(image.image_path) - create_parent_dirs(aux_abspath) - make_mip_from_zarr(zarr_path, aux_abspath, p_lower=args.p_lower, p_upper=args.p_upper) - logger.info(f"Wrote {aux_abspath}") + logger.trace(f"Creating auxiliary file: {aux_path}") + create_parent_dirs(aux_path) + store = fs.get_store(image.image_path) + make_mip_from_zarr(store, aux_path, p_lower=args.p_lower, p_upper=args.p_upper) + logger.info(f"Wrote {aux_path}") if args.thumbnail_name and (not metadata or not metadata.thumbnail_path): tb_path = get_aux_path(image.path, args.thumbnail_name) - tb_abspath = fs.get_absolute_path(tb_path) updated_obj['thumbnail_path'] = tb_path if fs.exists(tb_path): - logger.trace(f"Found thumbnail: {tb_abspath}") - elif aux_abspath: - logger.trace(f"Creating thumbnail: {tb_abspath}") - create_parent_dirs(tb_abspath) - make_thumbnail(aux_abspath, tb_abspath) - logger.info(f"Wrote {tb_abspath}") + logger.trace(f"Found thumbnail: {tb_path}") + elif aux_path: + logger.trace(f"Creating thumbnail: {tb_path}") + create_parent_dirs(tb_path) + make_thumbnail(aux_path, tb_path) + logger.info(f"Wrote {tb_path}") else: logger.trace(f"Cannot make thumbnail for {path} without aux image") diff --git a/docs/DataImport.md b/docs/DataImport.md new file mode 100644 index 0000000..5c962cf --- /dev/null +++ b/docs/DataImport.md @@ -0,0 +1,32 @@ +# Data Import + +You can import images into Zarrcade using the provided command line script: + +```bash +bin/import.py -d /root/data/dir -c mycollection +``` + +This will automatically create a local Sqlite database containing a Zarrcade **collection** named "mycollection" and populate it with information about the images in the specified directory. You can also add a label to the collection by setting the `--collection-label` parameter. This label will be displayed in the web UI when choosing the collection to view. + + +## Annotations + +You can add additional annotations to the images by providing a CSV file with the `-m` flag. The CSV file's first column must be a relative path to the OME-Zarr image within the root data directory. The remaining columns can be any annotations that will be searched and displayed within the gallery. + +You can modify the service configuration to control how the annotations are displayed and searched in the gallery. See the [Configuration](./Configuration.md) section for more details. + + +## Auxiliary Images + +By default, the import script will also create MIPs and thumbnails for each image in `./static/.zarrcade`. You can disable this by setting the `--no-aux` flag. You can change the output file location by setting the `--aux-path` parameter. Zarrcade will proxy the files automatically from within the `static` folder, but this may not be suitable for a production deployment. Instead, you can store the auxiliary images in the same directory as the OME-Zarr files. You will need to upload the files to your S3 bucket or other storage. Then, set `aux_image_mode: relative` in your `settings.yaml` to let Zarrcade know that your auxiliary files are stored relative to your data. + + +## Auxiliary Image Options + +Currently, Zarrcade supports creation of Maximum Intensity Projection (MIP) and thumbnails of the MIPs. You can control the brightness of the MIPs using the `--p-lower` and `--p-upper` parameters. + + +## Custom Auxiliary Image Generation + +You can create your own auxiliary images and thumbnails by setting the `--aux-path` parameter to the location where Zarrcade should find the files. Organize the folder structure to match the OME-Zarr files, and set the `aux_image_name` and `thumbnail_name` parameters to the names of the files containing the auxiliary images and thumbnails, respectively. During the import process, Zarrcade will detect that the files exist and will not attempt to generate them. + diff --git a/docs/settings.yaml.example b/docs/settings.yaml.example new file mode 100644 index 0000000..bb19177 --- /dev/null +++ b/docs/settings.yaml.example @@ -0,0 +1,24 @@ +log_level: INFO + +base_url: http://localhost:8000 + +database: + url: sqlite:///database.db + debug_sql: False + +filters: + - column_name: "Driver line" + data_type: string + filter_type: dropdown + - column_name: "Probes" + data_type: csv + filter_type: dropdown + - column_name: "Region" + - column_name: "Zoom" + +title_column_name: "Image Name" + +hide_columns: + - "Probes" + - "Image Name" + \ No newline at end of file diff --git a/templates/collection.html b/templates/collection.html index 4cbc65e..b5d13b1 100644 --- a/templates/collection.html +++ b/templates/collection.html @@ -27,7 +27,7 @@ {% if not dbimage.image_metadata.thumbnail_path %} Default thumbnail {% else %} - Image thumbnail + Image thumbnail {% endif %}
diff --git a/templates/details.html b/templates/details.html index fc67f9a..1a5930b 100644 --- a/templates/details.html +++ b/templates/details.html @@ -36,7 +36,7 @@

{{ get_title(dbimage) | safe }}

{% if dbimage.image_metadata and dbimage.image_metadata.aux_image_path %} - Image thumbnail + Image thumbnail {% else %} Default thumbnail {% endif %} @@ -54,7 +54,7 @@

{{ get_title(dbimage) | safe }}

Compression:{{ image.compression }} {% if dbimage.image_metadata %} {% for attr in column_map.keys() %} - {% if attr not in settings.details.hide_columns %} + {% if attr not in settings.hide_columns %} {{ column_map[attr] }}:{{ getattr(dbimage.image_metadata, attr) | safe }} {% endif %} {% endfor %} diff --git a/templates/form.html b/templates/form.html index 8f53704..2336bef 100644 --- a/templates/form.html +++ b/templates/form.html @@ -13,7 +13,7 @@ {% if filter.filter_type == FilterType.dropdown %}