diff --git a/carrot-test-config/carrot_commands.txt b/carrot-test-config/carrot_commands.txt index 10882a9..8d5bf3c 100644 --- a/carrot-test-config/carrot_commands.txt +++ b/carrot-test-config/carrot_commands.txt @@ -13,40 +13,40 @@ carrot_cli pipeline create --name "Example Pipeline" \ --description "Pipeline for demonstrating CARROT" # Create a template which defines the WDLs that will be used for any tests based on it -# Replace [pipeline_id] with the pipeline_id value returned by the `pipeline create` command -carrot_cli template create --pipeline_id [pipeline_id] --name "Example Template" \ +# Replace [pipeline_id_or_name] with the pipeline_id or name value returned by the `pipeline create` command +carrot_cli template create --pipeline [pipeline_id_or_name] --name "Example Template" \ --description "Template for demonstrating CARROT" \ --test_wdl https://raw.githubusercontent.com/broadinstitute/carrot-example-test/0.3.1-gamma/carrot-test-config/test.wdl \ --eval_wdl https://raw.githubusercontent.com/broadinstitute/carrot-example-test/0.3.1-gamma/carrot-test-config/eval.wdl # Map the template to the results you created, so the results will be accessible when the run # completes -# Replace [template_id] with the template_id returned by `template create` and replace the result -# ids with the result_id values from the `result create` command -carrot_cli template map_to_result [template_id] [first_result_id] test_workflow.data_file -carrot_cli template map_to_result [template_id] [second_result_id] eval_workflow.comparison_result +# Replace [template_id_or_name] with the template_id or name returned by `template create` and replace the result +# ids with the result_id_or_name values from the `result create` command +carrot_cli template map_to_result [template_id_or_name] [first_result_id_or_name] test_workflow.data_file +carrot_cli template map_to_result [template_id_or_name] [second_result_id_or_name] eval_workflow.comparison_result # Create a report for viewing your results all pretty when the run is complete carrot_cli report create --name "Example Report" --description "Report for demonstrating CARROT" \ --notebook report.ipynb # Map the template to the report so it will be generated automatically when the run completes -# Fill in the template_id from `template create` and the report_id from `report create` -carrot_cli template map_to_report [template_id] [report_id] +# Fill in the template_id_or_name from `template create` and the report_id_or_name from `report create` +carrot_cli template map_to_report [template_id_or_name] [report_id_or_name] # Create a test that defines default values that will be used for every run -# Fill in the template_id from `template create` -carrot_cli test create --name "Example Test" --template_id [template_id] \ +# Fill in the template_id_or_name from `template create` +carrot_cli test create --name "Example Test" --template [template_id_or_name] \ --description "Test for demonstrating CARROT" --test_input_defaults test_input_defaults.json \ --eval_input_defaults eval_input_defaults.json # Run the test -# Use the test_id from `test create` -carrot_cli test run --test_input test_input.json --eval_input eval_input.json [test_id] +# Use the test_id_or_name from `test create` +carrot_cli test run --test_input test_input.json --eval_input eval_input.json [test_id_or_name] # Get the status of the run you started, along with results if it has completed carrot_cli run find_by_id [run_id] # Get the status of the report for the run you started (once the run has completed), along with # the location of the report if it has completed -carrot_cli run find_report_by_ids [run_id] [report_id] \ No newline at end of file +carrot_cli run find_report_by_ids [run_id_or_name] [report_id_or_name] \ No newline at end of file diff --git a/carrot-test-config/report.ipynb b/carrot-test-config/report.ipynb index 88afbdd..f0e1f73 100644 --- a/carrot-test-config/report.ipynb +++ b/carrot-test-config/report.ipynb @@ -1,30 +1,4 @@ { - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - }, - "orig_nbformat": 2, - "kernelspec": { - "name": "python385jvsc74a57bd0aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49", - "display_name": "Python 3.8.5 64-bit" - }, - "metadata": { - "interpreter": { - "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2, "cells": [ { "cell_type": "code", @@ -32,9 +6,33 @@ "metadata": {}, "outputs": [], "source": [ - "# Control block\n", - "carrot_download_results = True\n", - "carrot_download_inputs = False" + "import csv\n", + "import os\n", + "import sys\n", + "\n", + "# Get results from the results file\n", + "results = {}\n", + "with open(\"results.csv\", newline='') as results_csv:\n", + " csv_reader = csv.reader(results_csv)\n", + " # Get the header\n", + " header = next(csv_reader)\n", + " # Now get the results\n", + " row = next(csv_reader)\n", + " for i in range(0, len(row)):\n", + " results[header[i]] = row[i]\n", + "# Download files in results\n", + "result_files = {}\n", + "for key, val in results.items():\n", + " if val.startswith('gs://'):\n", + " # Attempt to download with gsutil\n", + " os.makedirs(f'carrot_downloads/{key}', exist_ok=True)\n", + " filepath = f'carrot_downloads/{key.replace(\" \", \"_\")}/{val[val.rfind(\"/\")+1:]}'\n", + " download_status = os.system(f'gsutil cp {val} {filepath}')\n", + " # If it failed, print an error message and exit\n", + " if download_status != 0:\n", + " sys.exit(f\"gsutil terminated with an non-zero exit code when attempting to download {val}\")\n", + " # If it succeeded, add it to our file list\n", + " result_files[key] = filepath" ] }, { @@ -47,7 +45,7 @@ "\n", "# Load data and comparison files\n", "data = [[], []]\n", - "with open(carrot_downloads[\"results\"][\"example data file\"]) as data_file:\n", + "with open(result_files[\"example data file\"]) as data_file:\n", " data_csv = csv.reader(data_file)\n", " for row in data_csv:\n", " if len(row) > 1:\n", @@ -55,7 +53,7 @@ " data[1].append(float(row[1]))\n", "\n", "comparison = [[], []]\n", - "with open(carrot_downloads[\"results\"][\"example comparison file\"]) as comparison_file:\n", + "with open(result_files[\"example comparison file\"]) as comparison_file:\n", " comparison_csv = csv.reader(comparison_file)\n", " for row in comparison_csv:\n", " if len(row) > 1:\n", @@ -98,5 +96,35 @@ "plt.show()" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "interpreter": { + "hash": "1ee38ef4a5a9feb55287fd749643f13d043cb0a7addaab2a9c224cbe137c0062" + }, + "kernelspec": { + "display_name": "Python 3.8.12 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "metadata": { + "interpreter": { + "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" + } + }, + "orig_nbformat": 2 + }, + "nbformat": 4, + "nbformat_minor": 2 +}