Merge pull request #1 from broadinstitute/kl_changes_for_carrot_0.5

Modified the report notebook to reflect changes to reporting in carrot 0.5.0 and commands to reflect changes in carrot 0.4.0
broadinstitute · Jun 7, 2022 · b05fabb · b05fabb
2 parents c34b3cc + 10dba36
commit b05fabb
Show file tree

Hide file tree

Showing 2 changed files with 74 additions and 46 deletions.
diff --git a/carrot-test-config/carrot_commands.txt b/carrot-test-config/carrot_commands.txt
@@ -13,40 +13,40 @@ carrot_cli pipeline create --name "Example Pipeline" \
 --description "Pipeline for demonstrating CARROT"
 
 # Create a template which defines the WDLs that will be used for any tests based on it
-# Replace [pipeline_id] with the pipeline_id value returned by the `pipeline create` command
-carrot_cli template create --pipeline_id [pipeline_id] --name "Example Template" \
+# Replace [pipeline_id_or_name] with the pipeline_id or name value returned by the `pipeline create` command
+carrot_cli template create --pipeline [pipeline_id_or_name] --name "Example Template" \
 --description "Template for demonstrating CARROT" \
 --test_wdl https://raw.githubusercontent.com/broadinstitute/carrot-example-test/0.3.1-gamma/carrot-test-config/test.wdl \
 --eval_wdl https://raw.githubusercontent.com/broadinstitute/carrot-example-test/0.3.1-gamma/carrot-test-config/eval.wdl
 
 # Map the template to the results you created, so the results will be accessible when the run 
 # completes
-# Replace [template_id] with the template_id returned by `template create` and replace the result
-# ids with the result_id values from the `result create` command
-carrot_cli template map_to_result [template_id] [first_result_id] test_workflow.data_file
-carrot_cli template map_to_result [template_id] [second_result_id] eval_workflow.comparison_result
+# Replace [template_id_or_name] with the template_id or name returned by `template create` and replace the result
+# ids with the result_id_or_name values from the `result create` command
+carrot_cli template map_to_result [template_id_or_name] [first_result_id_or_name] test_workflow.data_file
+carrot_cli template map_to_result [template_id_or_name] [second_result_id_or_name] eval_workflow.comparison_result
 
 # Create a report for viewing your results all pretty when the run is complete
 carrot_cli report create --name "Example Report" --description "Report for demonstrating CARROT" \
 --notebook report.ipynb
 
 # Map the template to the report so it will be generated automatically when the run completes
-# Fill in the template_id from `template create` and the report_id from `report create`
-carrot_cli template map_to_report [template_id] [report_id]
+# Fill in the template_id_or_name from `template create` and the report_id_or_name from `report create`
+carrot_cli template map_to_report [template_id_or_name] [report_id_or_name]
 
 # Create a test that defines default values that will be used for every run
-# Fill in the template_id from `template create`
-carrot_cli test create --name "Example Test" --template_id [template_id] \
+# Fill in the template_id_or_name from `template create`
+carrot_cli test create --name "Example Test" --template [template_id_or_name] \
 --description "Test for demonstrating CARROT" --test_input_defaults test_input_defaults.json \
 --eval_input_defaults eval_input_defaults.json
 
 # Run the test
-# Use the test_id from `test create`
-carrot_cli test run --test_input test_input.json --eval_input eval_input.json [test_id]
+# Use the test_id_or_name from `test create`
+carrot_cli test run --test_input test_input.json --eval_input eval_input.json [test_id_or_name]
 
 # Get the status of the run you started, along with results if it has completed
 carrot_cli run find_by_id [run_id]
 
 # Get the status of the report for the run you started (once the run has completed), along with
 # the location of the report if it has completed
-carrot_cli run find_report_by_ids [run_id] [report_id]
+carrot_cli run find_report_by_ids [run_id_or_name] [report_id_or_name]
diff --git a/carrot-test-config/report.ipynb b/carrot-test-config/report.ipynb
@@ -1,40 +1,38 @@
 {
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  },
-  "orig_nbformat": 2,
-  "kernelspec": {
-   "name": "python385jvsc74a57bd0aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49",
-   "display_name": "Python 3.8.5 64-bit"
-  },
-  "metadata": {
-   "interpreter": {
-    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2,
  "cells": [
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Control block\n",
-    "carrot_download_results = True\n",
-    "carrot_download_inputs = False"
+    "import csv\n",
+    "import os\n",
+    "import sys\n",
+    "\n",
+    "# Get results from the results file\n",
+    "results = {}\n",
+    "with open(\"results.csv\", newline='') as results_csv:\n",
+    "    csv_reader = csv.reader(results_csv)\n",
+    "    # Get the header\n",
+    "    header = next(csv_reader)\n",
+    "    # Now get the results\n",
+    "    row = next(csv_reader)\n",
+    "    for i in range(0, len(row)):\n",
+    "        results[header[i]] = row[i]\n",
+    "# Download files in results\n",
+    "result_files = {}\n",
+    "for key, val in results.items():\n",
+    "    if val.startswith('gs://'):\n",
+    "        # Attempt to download with gsutil\n",
+    "        os.makedirs(f'carrot_downloads/{key}', exist_ok=True)\n",
+    "        filepath = f'carrot_downloads/{key.replace(\" \", \"_\")}/{val[val.rfind(\"/\")+1:]}'\n",
+    "        download_status = os.system(f'gsutil cp {val} {filepath}')\n",
+    "        # If it failed, print an error message and exit\n",
+    "        if download_status != 0:\n",
+    "            sys.exit(f\"gsutil terminated with an non-zero exit code when attempting to download {val}\")\n",
+    "        # If it succeeded, add it to our file list\n",
+    "        result_files[key] = filepath"
    ]
   },
   {
@@ -47,15 +45,15 @@
     "\n",
     "# Load data and comparison files\n",
     "data = [[], []]\n",
-    "with open(carrot_downloads[\"results\"][\"example data file\"]) as data_file:\n",
+    "with open(result_files[\"example data file\"]) as data_file:\n",
     "    data_csv = csv.reader(data_file)\n",
     "    for row in data_csv:\n",
     "        if len(row) > 1:\n",
     "            data[0].append(float(row[0]))\n",
     "            data[1].append(float(row[1]))\n",
     "\n",
     "comparison = [[], []]\n",
-    "with open(carrot_downloads[\"results\"][\"example comparison file\"]) as comparison_file:\n",
+    "with open(result_files[\"example comparison file\"]) as comparison_file:\n",
     "    comparison_csv = csv.reader(comparison_file)\n",
     "    for row in comparison_csv:\n",
     "        if len(row) > 1:\n",
@@ -98,5 +96,35 @@
     "plt.show()"
    ]
   }
- ]
-}
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "1ee38ef4a5a9feb55287fd749643f13d043cb0a7addaab2a9c224cbe137c0062"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.8.12 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.13"
+  },
+  "metadata": {
+   "interpreter": {
+    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
+   }
+  },
+  "orig_nbformat": 2
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}