hubmapconsortium · jpuerto-psc · Sep 12, 2024 · Aug 16, 2024 · Aug 28, 2024 · Sep 4, 2024
diff --git a/README.md b/README.md
@@ -7,6 +7,15 @@ We welcome contributions! Please see our [contributing guidelines](https://githu
 
 We also have a slack channel for feedback on the workspaces. Please reach out to the HuBMAP Help Desk ([[email protected]](mailto:[email protected])) to get in touch.
 
+## Deploying User Templates Locally
+- Clone the repository.
+- Generate a virtual environment (`virtualenv -p python3.9 venv`).
+- Activate the virtual environment (`source venv/bin/activate`).
+- Install the requirements (`pip install -r requirements.txt`).
+- Create a config file (`cp src/example_config.json src/config.json`) and update it with appropriate values.
+- Run database migration (`python src/manage.py migrate`).
+- Start the server (`python src/manage.py runserver`).
+
 
 ## Contributors
 This project is part of the HuBMAP consortium. The main contributors to the workspaces are the [Pittsburgh Supercomputing Center](https://www.psc.edu/) and the [HIDIVE Lab](https://hidivelab.org) at [Harvard Medical School](https://hms.harvard.edu).
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.7.0
+1.8.0
diff --git a/src/example_config.json b/src/example_config.json
@@ -1,17 +1,23 @@
 {
+  "SECRET_KEY": "secret_key_here",
+  "DEBUG": true,
+  "ALLOWED_HOSTS": [
+    "your_host"
+  ],
   "template_types": {
     "jupyter_lab": {
       "usw_job_type": "",
       "description": ""
     }
   },
-  "PORTAL_UI_BASE": "",
-  "ENTITY_API_BASE": "",
-  "ELASTICSEARCH_ENDPOINT": "",
-  "PORTAL_INDEX_PATH": "",
-  "SOFT_ASSAY_ENDPOINT": "",
-  "SOFT_ASSAY_ENDPOINT_PATH": "",
-  "ASSETS_ENDPOINT": "",
-  "GLOBUS_CLIENT_ID": "",
-  "GLOBUS_CLIENT_SECRET": ""
-}
+  "PORTAL_UI_BASE": "https://portal.hubmapconsortium.org",
+  "ENTITY_API_BASE": "https://entity.api.hubmapconsortium.org",
+  "ELASTICSEARCH_ENDPOINT": "https://search.api.hubmapconsortium.org",
+  "PORTAL_INDEX_PATH": "/v3/portal/search",
+  "UBKG_ENDPOINT": "https://ontology.api.hubmapconsortium.org",
+  "SOFT_ASSAY_ENDPOINT": "https://ingest.api.hubmapconsortium.org",
+  "SOFT_ASSAY_ENDPOINT_PATH": "assaytype",
+  "ASSETS_ENDPOINT": "https://assets.hubmapconsortium.org",
+  "GLOBUS_CLIENT_ID": "YOUR_CLIENT_ID",
+  "GLOBUS_CLIENT_SECRET": "YOUR_CLIENT_SECRET"
+}
diff --git a/src/user_templates_api/templates/jupyter_lab/templates/api_tutorial/metadata.json b/src/user_templates_api/templates/jupyter_lab/templates/api_tutorial/metadata.json
@@ -7,6 +7,13 @@
   ],
   "is_multi_dataset_template": true,
   "template_format": "python",
+  "examples": [
+    {
+      "title": "Working with API examples",
+      "description": "Example metadata and available files for four datasets.",
+      "datasets": ["41d665c06bbec9b4dcaec6d835acb0fb", "03a8a12acfa25c2414cfdada4a4b5fd6", "880474c3108655acfe6f8c30137f22ae", "3bc3ad124014a632d558255626bf38c9"]
+    }
+  ],
   "is_hidden": false,
   "last_modified_unix_timestamp": 1723749414
 }
diff --git a/src/user_templates_api/templates/jupyter_lab/templates/blank/metadata.json b/src/user_templates_api/templates/jupyter_lab/templates/blank/metadata.json
@@ -5,6 +5,13 @@
     ], 
     "is_multi_dataset_template": true,
     "template_format": "jinja",
+    "examples": [
+      {
+        "title": "Workspace Tutorial datasets",
+        "description": "Create a blank notebook with a CODEX and Slideseq dataset linked.",
+        "datasets": ["69c70762689b20308bb049ac49653342", "a1d17fdd270a69c813b872a927dfa5f3"]
+      }
+    ],
     "is_hidden": false,
     "last_modified_unix_timestamp": 1723749414
 }
diff --git a/src/user_templates_api/templates/jupyter_lab/templates/celltypes_salmon/metadata.json b/src/user_templates_api/templates/jupyter_lab/templates/celltypes_salmon/metadata.json
@@ -8,6 +8,14 @@
     ],
     "is_multi_dataset_template": true,
     "template_format": "jinja",
+    "examples": [
+      {
+        "title": "Example cell types of kidney datasets",
+        "description": "Create a few interactive visualizations to example kidney celltype compositions.",
+        "required_filetypes": ["secondary_analysis.h5ad"],
+        "datasets": ["a1d17fdd270a69c813b872a927dfa5f3", "cdf9127efdfc96189845c35244d4d8ab", "fcd7a8d45be34fdccb8fa400ea81a331"]
+      }
+    ],
     "is_hidden": false,
     "last_modified_unix_timestamp": 1723749414
   }
diff --git a/src/user_templates_api/templates/jupyter_lab/templates/celltypes_salmon/template.txt b/src/user_templates_api/templates/jupyter_lab/templates/celltypes_salmon/template.txt
@@ -15,7 +15,7 @@
    "outputs": [],
    "source": [
     "# !pip install --upgrade pip\n",
-    "# !pip install numpy pandas requests wheel matplotlib matplotlib-inline altair seaborn anndata"
+    "# !pip install numpy pandas requests wheel matplotlib matplotlib-inline altair seaborn anndata hubmap-template-helper"
    ]
   },
   {
@@ -40,7 +40,10 @@
     "import seaborn as sns\n",
     "import altair as alt\n",
     "\n",
-    "import anndata as ad"
+    "import anndata as ad\n",
+    "\n",
+    "from hubmap_template_helper import uuids as hth_uuids\n",
+    "from hubmap_template_helper import compatibility as hth_comp"
    ]
   },
   {
@@ -60,7 +63,7 @@
    "metadata": {},
    "source": [
     "## Linked datasets\n",
-    "The following datasets were symlinked to the workspace when this template was added:"
+    "The following datasets were linked when the workspace was created."
    ]
   },
   {
@@ -73,7 +76,7 @@
     "uuids = {{ uuids | safe }}\n",
     "\n",
     "# required file_types\n",
-    "required_filestypes = ['secondary_analysis.h5ad']\n",
+    "required_filetypes = ['secondary_analysis.h5ad']\n",
     "\n",
     "# search_api\n",
     "search_api = 'https://search.api.hubmapconsortium.org/v3/portal/search'"
@@ -83,93 +86,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The following checks if the datasets are compatible with this template."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# This template is created for particular datatypes only.\n",
-    "# This functions checks for each uuids above whether they have the correct datatypes.\n",
-    "\n",
-    "def check_template_compatibility(uuids, accepted_datatypes=None, required_filetypes=None, search_api = 'https://search.api.hubmapconsortium.org/v3/portal/search'): \n",
-    "    '''\n",
-    "    For a set of HuBMAP UUIDs, check if valid, and return valid UUIDs.\n",
-    "    Checks if UUIDs are present in the search API. \n",
-    "    If accepted_datatypes is defined, checks if UUIDs are of any of the datatypes in accepted_datatypes.\n",
-    "    If required_filetypes is defined, checks if UUIDs have all of the required filetypes in required_filetypes.\n",
-    "\n",
-    "    Parameters\n",
-    "    ----------\n",
-    "    uuids : array of string\n",
-    "        HuBMAP UUIDs to be checked\n",
-    "    accepted_datatypes: array of string, optional\n",
-    "        accepted datatypes for template\n",
-    "    required_filetypes: array of string, optional\n",
-    "        required datatypes for template\n",
-    "    search_api: string, optional\n",
-    "        URL of search API\n",
-    "\n",
-    "    Returns\n",
-    "    -------\n",
-    "    array of string\n",
-    "        valid UUIDs\n",
-    "    '''\n",
-    "    hits = json.loads(\n",
-    "        requests.post(\n",
-    "            search_api,\n",
-    "            json={\n",
-    "                'size': 10000,\n",
-    "                'query': {'ids': {'values': uuids}},\n",
-    "                '_source': ['files', 'data_types']\n",
-    "            }, \n",
-    "        ).text\n",
-    "    )['hits']['hits']\n",
-    "\n",
-    "    # create mapping for uuid to file_types and data_types\n",
-    "    uuid_to_files = {}\n",
-    "    uuid_to_datatypes = {}\n",
-    "    for hit in hits:\n",
-    "        file_paths = [file['rel_path'] for file in hit['_source']['files']]\n",
-    "        uuid_to_files[hit['_id']] = file_paths\n",
-    "\n",
-    "        hit_data_type = hit['_source']['data_types']\n",
-    "        uuid_to_datatypes[hit['_id']] = hit_data_type\n",
-    "    \n",
-    "    # save uuids without warnings\n",
-    "    accepted_uuids = uuids.copy()\n",
-    "\n",
-    "    # remove unvalid uuids\n",
-    "    for uuid in uuids: \n",
-    "        # check if all uuids are found in the search api\n",
-    "        if uuid not in uuid_to_files.keys(): \n",
-    "            warnings.warn('Dataset with UUID \"' + uuid + '\" not found in Search API')\n",
-    "            accepted_uuids.remove(uuid)\n",
-    "            continue\n",
-    "\n",
-    "        if required_filetypes is not None: \n",
-    "            # check if file_types for each uuid are in required_filetypes\n",
-    "            file_types = uuid_to_files[uuid]\n",
-    "            for required_file_type in required_filetypes:\n",
-    "                if required_file_type not in file_types:\n",
-    "                    warnings.warn('Dataset with UUID \"' + uuid + '\" does not have required file type: ' + required_file_type)\n",
-    "                    if uuid in accepted_uuids:\n",
-    "                        accepted_uuids.remove(uuid)\n",
-    "\n",
-    "        if accepted_datatypes is not None: \n",
-    "            # check if data_types for each uuid are in accepted_datatypes\n",
-    "            data_types = uuid_to_datatypes[uuid]\n",
-    "            for data_type in data_types:\n",
-    "                if data_type not in accepted_datatypes: \n",
-    "                    warnings.warn('Dataset with UUID \"' + uuid + '\" has unaccepted data type: ' + data_type)\n",
-    "                    if uuid in accepted_uuids:\n",
-    "                        accepted_uuids.remove(uuid)\n",
-    "                    continue\n",
-    "    \n",
-    "    return accepted_uuids"
+    "This template is created for particular data types only. The following checks if the datasets are compatible with this template."
    ]
   },
   {
@@ -178,7 +95,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "uuids = check_template_compatibility(uuids, required_filetypes=required_filestypes, search_api=search_api)"
+    "uuids = hth_comp.check_template_compatibility(uuids, required_filetypes=required_filetypes, search_api=search_api)"
    ]
   },
   {
@@ -196,31 +113,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def get_uuid_to_hubmap(uuids): \n",
-    "    '''\n",
-    "    Retrieve a dictionary mapping uuids to hubmap ids.\n",
-    "\n",
-    "    Parameters\n",
-    "    ----------\n",
-    "    uuids : list\n",
-    "        list with uuids\n",
-    "    \n",
-    "    Returns\n",
-    "    -------\n",
-    "    dictionary mapping uuids to hubmap ids\n",
-    "    '''\n",
-    "    ## Fetch metadata, and read it into a dataframe\n",
-    "    response = requests.post(\n",
-    "        'https://portal.hubmapconsortium.org/metadata/v0/datasets.tsv', json={'uuids': uuids}\n",
-    "    )\n",
-    "    metadata = list(DictReader(StringIO(response.text), dialect=excel_tab))\n",
-    "    metadata = pd.DataFrame(metadata[1:])\n",
-    "\n",
-    "    ## Create mapping from uuid to hubmap id\n",
-    "    uuid_to_hubmap = dict(zip(metadata['uuid'], metadata['hubmap_id']))\n",
-    "    return uuid_to_hubmap\n",
-    "\n",
-    "uuid_to_hubmap = get_uuid_to_hubmap(uuids)\n",
+    "uuid_to_hubmap = hth_uuids.get_uuid_to_hubmap_mapping(uuids)\n",
     "uuid_to_hubmap"
    ]
   },
@@ -298,7 +191,7 @@
     "\n",
     "    all_clid = None\n",
     "\n",
-    "    uuid_to_hubmap = get_uuid_to_hubmap(uuids)\n",
+    "    uuid_to_hubmap = hth_uuids.get_uuid_to_hubmap_mapping(uuids)\n",
     "\n",
     "    # can't loop over a single uuid \n",
     "    if(type(uuids) == str):\n",
@@ -346,13 +239,6 @@
     "adata_i = ad.read_h5ad('./datasets/' + uuids[1] + '/secondary_analysis.h5ad')"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "attachments": {},
    "cell_type": "markdown",

diff --git a/src/user_templates_api/templates/jupyter_lab/templates/compress_anndata/metadata.json b/src/user_templates_api/templates/jupyter_lab/templates/compress_anndata/metadata.json
@@ -6,6 +6,13 @@
   ],
   "is_multi_dataset_template": true,
   "template_format": "jinja",
+  "examples": [
+    {
+      "title": "Compress kidney h5ad file",
+      "description": "Use a h5ad file to retrieve anndata, remove all but X, obs and var, and save it.",
+      "datasets": ["0db9481caed92aec4cddc606108692d5"]
+    }
+  ],
   "is_hidden": false,
-    "last_modified_unix_timestamp": 1723749414
+  "last_modified_unix_timestamp": 1723749414
 }
diff --git a/src/user_templates_api/templates/jupyter_lab/templates/file_retrieval/metadata.json b/src/user_templates_api/templates/jupyter_lab/templates/file_retrieval/metadata.json
@@ -1,12 +1,19 @@
 {
-    "title": "Retrieve files with the search and assets API",
-    "description": "Use the search API to get information about datasets and use the assets API to retrieve files.",
-    "tags": [
-        "files",
-        "api"
-    ], 
-    "is_multi_dataset_template": true,
-    "template_format": "jinja",
-    "is_hidden": false,
-    "last_modified_unix_timestamp": 1723749414
+  "title": "Retrieve files with the search and assets API",
+  "description": "Use the search API to get information about datasets and use the assets API to retrieve files.",
+  "tags": [
+      "files",
+      "api"
+  ], 
+  "is_multi_dataset_template": true,
+  "template_format": "jinja",
+  "examples": [
+    {
+      "title": "Retrieve a full dataset",
+      "description": "Use the assets API to retrieve a full seqFISH dataset.",
+      "datasets": ["c6a254b2dc2ed46b002500ade163a7cc"]
+    }
+  ],
+  "is_hidden": false,
+  "last_modified_unix_timestamp": 1723749414
 }
diff --git a/src/user_templates_api/templates/jupyter_lab/templates/new_template/README.md b/src/user_templates_api/templates/jupyter_lab/templates/new_template/README.md
@@ -15,12 +15,14 @@ The README contains further information about the template. At least include 1 d
 
 ## metadata.json
 The metadata.json is used to display the templates in the HuBMAP Data Portal. It consists of: 
-- title: a title of the notebook.
-- description: a 1-2 sentence description of the notebook.
-- tags: a list of tags to describe the notebook. Potential tags can be found [here](https://github.com/hubmapconsortium/user-templates-api/tree/development/src/tags.json). New tags can be added if necessary. 
-- is_multi_dataset_template: boolean describing whether template can handle more than 1 dataset
-- template_format: one of python/json/jinja. We recommend using jinja, and describe the insertion of cells for jinja below.
-- is_hidden: Any template with this field set to `true` is not shown in the Portal.
+- `title`: a title of the notebook.
+- `description`: a 1-2 sentence description of the notebook.
+- `tags`: a list of tags to describe the notebook. Potential tags can be found [here](https://github.com/hubmapconsortium/user-templates-api/tree/development/src/tags.json). New tags can be added if necessary. 
+- `is_multi_dataset_template`: boolean describing whether template can handle more than 1 dataset.
+- `template_format`: one of `python`/`json`/`jinja`. We recommend using `jinja`, and describe the insertion of cells for jinja below.
+- `examples`: array with examples that are shown in the template example pages. Each example has a `title` (string), a `description` (string), and `datasets` (array of strings). Optionally, it also has an `assay_display_name` (array) with the possible assay_display_names, and `required_filetypes` (array) with required filetypes.
+- `is_hidden`: Any template with this field set to `true` is not shown in the Portal.
+- `last_modified_unix_timestamp`: Don't set this, it is added automatically to keep track of changes.
 
 
 ## render.py
@@ -33,3 +35,13 @@ The actual template. This can be directly converted from .ipynb to .txt, or use
 uuids = {{ uuids | safe }}
 ```
 An example can be found in the blank template.
+
+
+## Helper package
+A few functions that help with creating templates, such as a template compatibility checker, are included in a small helper package. Install it as such:
+
+```sh
+pip install hubmap-template-helper
+```
+
+Read more [here](https://github.com/thomcsmits/hubmap_template_helper).