Merge branch 'master' into docs

mskcc · Apr 7, 2022 · 7715ad9 · 7715ad9
2 parents 05aefe0 + 07e7823
commit 7715ad9
Show file tree

Hide file tree

Showing 119 changed files with 1,827 additions and 3,491 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,3 +24,5 @@ worker_log.txt
 toil_runs/
 tmp*
 run-toil.sh
+old/
+cwltool_runs/
diff --git a/Makefile b/Makefile
@@ -48,6 +48,11 @@ make test
 
 # run tests in parallel
 make test3 -j 8
+
+# or
+
+# run integration tests only
+make integration_test
 ```
 
 To do a version bump of all CWL files to a new version of the helix_filters container, use:
@@ -115,78 +120,78 @@ init:
 export SINGULARITY_CACHEDIR:=/juno/work/ci/pluto-cwl-test/cache
 # GIT_TAG:=$(shell git describe --tags --abbrev=0)
 HF_CONTAINER:=mskcc/helix_filters_01
-HF_TAG:=21.3.4
+HF_TAG:=21.3.5
 DOCKER_TAG:=$(HF_CONTAINER):$(HF_TAG)
 DOCKER_DEV_TAG:=$(HF_CONTAINER):latest
 # NOTE: you cannot use a filename with a ':' as a Makefile target
 SINGULARITY_SIF:=mskcc_helix_filters_01:$(HF_TAG).sif
 SINGULARITY_DEV_SIF:=mskcc_helix_filters_01:latest.sif
 singularity-pull:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(SINGULARITY_SIF)" docker://$(DOCKER_TAG)
 
 singularity-pull-dev:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(SINGULARITY_DEV_SIF)" docker://$(DOCKER_DEV_TAG)
 # unset SINGULARITY_CACHEDIR && \
 # module load singularity/3.3.0 && \
 
 # shell into the Singularity container to check that it looks right
 singularity-shell:
-	- . "$(ENVSH)" singularity && \
+	- . "$(ENVSH)" toil && \
 	singularity shell "$(SINGULARITY_SIF)"
 
 # mskcc/roslin-variant-facets:1.6.3
 # mskcc/helix_filters_01:facets-1.6.3
 FACETS_DOCKERTAG:=mskcc/helix_filters_01:facets-1.6.3
 FACETS_SIF:=mskcc_helix_filters_01:facets-1.6.3.sif
 singularity-pull-facets:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(FACETS_SIF)" docker://$(FACETS_DOCKERTAG)
 
 FACETS_SUITE_DOCKERTAG:=mskcc/helix_filters_01:facets-suite-2.0.6
 FACETS_SUITE_SIF:=helix_filters_01_facets-suite-2.0.6.sif
 singularity-pull-facets-suite:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(FACETS_SUITE_SIF)" docker://$(FACETS_SUITE_DOCKERTAG)
 
 # docker://cmopipeline/getbasecountsmultisample:1.2.2
 FILLOUT_DOCKERTAG:=mskcc/helix_filters_01:getbasecountsmultisample-1.2.2
 FILLOUT_SIF:=helix_filters_01_getbasecountsmultisample-1.2.2.sif
 singularity-pull-fillout:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(FILLOUT_SIF)" docker://$(FILLOUT_DOCKERTAG)
 
 MSI_DOCKERTAG:=mskcc/msisensor:0.2
 MSI_SIF:=mskcc_msisensor:0.2.sif
 singularity-pull-msi:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(MSI_SIF)" docker://$(MSI_DOCKERTAG)
 
 IGV_REPORTS_DOCKERTAG:=$(HF_CONTAINER):igv-reports-1.0.1
 IGV_REPORTS_SIF:=mskcc_helix_filters_01:igv-reports-1.0.1.sif
 singularity-pull-igv-reports:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(IGV_REPORTS_SIF)" docker://$(IGV_REPORTS_DOCKERTAG)
 
 # mskcc/roslin-variant-cmo-utils:1.9.15
 # mskcc_roslin-variant-cmo-utils:1.9.15.sif
 CMOUTILS_DOCKERTAG:=mskcc/roslin-variant-cmo-utils:1.9.15
 CMOUTILS_SIF:=mskcc_roslin-variant-cmo-utils:1.9.15.sif
 singularity-pull-cmoutils:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(CMOUTILS_SIF)" docker://$(CMOUTILS_DOCKERTAG)
 
 R_DOCKERTAG:=mskcc/helix_filters_01:R-3.5.1
 R_SIF:=mskcc_helix_filters_01:R-3.5.1.sif
 singularity-pull-r:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(R_SIF)" docker://$(R_DOCKERTAG)
 
 REPORT_DOCKERTAG:=mskcc/helix_filters_01:reporting
 REPORT_SIF:=mskcc_helix_filters_01:reporting.sif
 singularity-pull-report:
-	. "$(ENVSH)" singularity && \
+	. "$(ENVSH)" toil && \
 	singularity pull --force --name "$(REPORT_SIF)" docker://$(REPORT_DOCKERTAG)
 
 
@@ -201,7 +206,7 @@ singularity-pull-vep:
 	if [ -e "$(VEP_SIF_LOCAL)" ]; then
 	ln -s "$(VEP_SIF_LOCAL)" "$(VEP_SIF)"
 	else
-	. "$(ENVSH)" singularity && singularity pull --force --name "$(VEP_SIF)" docker://$(VEP_DOCKERTAG)
+	. "$(ENVSH)" toil && singularity pull --force --name "$(VEP_SIF)" docker://$(VEP_DOCKERTAG)
 	fi
 	fi
 # rsync -vrthP "$(VEP_SIF_LOCAL)" "$(VEP_SIF)"
@@ -227,32 +232,48 @@ export FIXTURES_DIR:=/juno/work/ci/helix_filters_01/fixtures
 
 # TODO: figure out why this is missing some tests
 test2:
-	. "$(ENVSH)" test && \
+	. "$(ENVSH)" toil && \
 	if [ ! -e "$(SINGULARITY_SIF)" ]; then $(MAKE) singularity-pull; fi && \
 	python3 test.py
 
 # TODO: figure out if we can run the tests in parallel or otherwise make it faster
 # for some reason the test recipe is not running all tests....
 test:
-	. "$(ENVSH)" test && \
+	. "$(ENVSH)" toil && \
 	if [ ! -e "$(SINGULARITY_SIF)" ]; then $(MAKE) singularity-pull; fi && \
 	for i in tests/test_*.py; do echo $$i; python3 $$i; done
 
 # run tests in parallel;
 # $ make test3 -j 4
+# the test
+TEST_ENV:=toil
 TESTS:=$(shell ls tests/test_*.py)
 $(TESTS):
-	. "$(ENVSH)" test && echo $@; python3 $@
+	. "$(ENVSH)" "$(TEST_ENV)" && echo $@; python3 $@
 .PHONY:$(TESTS)
 test3:$(TESTS)
 
-# run the integration tests with Jenkins
+
 integration_test:
-	. "$(ENVSH)" integration_test
-	python tests/test_workflow_with_facets.xl.py
+	. "$(ENVSH)" integration_test && \
+	cd pluto && \
+	python test_tools.py && \
+	python test_serializer.py && \
+	cd .. && \
+	#for i in tests/test_*workflow*.py; do echo $$i; python3 $$i; rm -rf $TMP_DIR/tmp* /scratch/jenkins/tmp*; done
+	python tests/test_workflow_with_facets.xl.py && \
 	python tests/test_workflow_with_facets.medium.py
-	python tests/test_samples_fillout_index_workflow_cwl.py
-	( cd pluto && python test_tools.py && python test_serializer.py )
+# run the integration tests with Jenkins
+# integration_test:
+# 	. "$(ENVSH)" integration_test
+# 	python tests/test_workflow_with_facets.xl.py
+# 	python tests/test_workflow_with_facets.medium.py
+# 	python tests/test_samples_fillout_index_workflow_cwl.py
+# 	( cd pluto && python test_tools.py && python test_serializer.py )
+
+integration_test_1:
+	. "$(ENVSH)" integration_test && \
+	for i in tests/test_*cwl*.py; do echo $$i; python3 $$i; done
 
 # interactive session with environment populated
 bash: ENV=shell

diff --git a/README.md b/README.md
@@ -142,3 +142,5 @@ python tests/test_workflow_cwl.py TestClassName
 
 python tests/test_workflow_cwl.py TestClassName.test_function
 ```
+
+
diff --git a/cwl/add_af.cwl b/cwl/add_af.cwl
@@ -7,7 +7,7 @@ baseCommand: ['add_af.py']
 
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 
 inputs:
   input_file:

diff --git a/cwl/add_is_in_impact.cwl b/cwl/add_is_in_impact.cwl
@@ -7,7 +7,7 @@ baseCommand: ['add_is_in_impact.py', '--include-assay']
 
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 
 inputs:
   input_file:

diff --git a/cwl/add_msi_status.cwl b/cwl/add_msi_status.cwl
@@ -5,7 +5,7 @@ class: CommandLineTool
 baseCommand: [ "add_msi_status.py" ]
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 
 inputs:
   input_filename:

diff --git a/cwl/annotate-maf-wrapper.cwl b/cwl/annotate-maf-wrapper.cwl
@@ -5,26 +5,21 @@
 # --facets-output $${facets_rds} \
 # -o $${annot_maf}
 
-cwlVersion: v1.0
+cwlVersion: v1.2
 class: CommandLineTool
 successCodes: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]
 stdout: annotate_maf_stdout.txt
 stderr: annotate_maf_stderr.txt
-baseCommand: ["bash","run_annotate_maf_wrapper.sh"]
+baseCommand: ["annotate-maf-wrapper.R","--parallel","TRUE"]
 
 requirements:
+  InlineJavascriptRequirement: {}
   ResourceRequirement:
     ramMin: 8000
     coresMin: 4
     # annotate-maf-wrapper.R does not actually accept an arg for the number of CPUs to use, it defaults to using all available CPUs
   DockerRequirement:
     dockerPull: mskcc/helix_filters_01:facets-suite-2.0.6
-  InitialWorkDirRequirement:
-    listing:
-      - entryname: run_annotate_maf_wrapper.sh
-        entry: |-
-          annotate-maf-wrapper.R --parallel TRUE $@ || touch failed.txt
-
 inputs:
   maf_file:
     type: File
@@ -47,10 +42,13 @@ outputs:
     type: File?
     outputBinding:
       glob: $(inputs.output_filename) # $${pair_id}_hisens.ccf.maf
-  failed_txt:
-    type: File?
+  failed:
+    type: boolean
     outputBinding:
-      glob: failed.txt
+      outputEval: |
+        ${
+          return runtime.exitCode != 0;
+        }
   stdout_txt:
     type: stdout
   stderr_txt:

diff --git a/cwl/calc-tmb.cwl b/cwl/calc-tmb.cwl
@@ -6,7 +6,7 @@ baseCommand: ["calc-tmb.py", 'from-file']
 
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 
 inputs:
   input_file:

diff --git a/cwl/cat.cwl b/cwl/cat.cwl
@@ -5,7 +5,7 @@ class: CommandLineTool
 baseCommand: [ cat ]
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 stdout: output.txt
 inputs:
   input_files:

diff --git a/cwl/concat-mafs.cwl b/cwl/concat-mafs.cwl
@@ -39,7 +39,7 @@ baseCommand: [
 requirements:
   InlineJavascriptRequirement: {}
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
   InitialWorkDirRequirement:
     listing:
       - entryname: inputs_dir

diff --git a/cwl/concat-mafs_all_cols.cwl b/cwl/concat-mafs_all_cols.cwl
@@ -16,7 +16,7 @@ baseCommand: [
 requirements:
   InlineJavascriptRequirement: {}
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
   InitialWorkDirRequirement:
     listing:
       - entryname: inputs_dir

diff --git a/cwl/concat-tables.cwl b/cwl/concat-tables.cwl
@@ -5,7 +5,7 @@ class: CommandLineTool
 baseCommand: [ "concat-tables.py" ]
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 inputs:
   output_filename:
     type: string

diff --git a/cwl/concat-tables_dir.cwl b/cwl/concat-tables_dir.cwl
@@ -8,7 +8,7 @@ doc: "Concatenate all the table files provided, but put the input files in a dir
 requirements:
   InlineJavascriptRequirement: {}
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
   InitialWorkDirRequirement:
     listing:
       - entryname: inputs_dir

diff --git a/cwl/concat.cwl b/cwl/concat.cwl
@@ -10,7 +10,7 @@ requirements:
   StepInputExpressionRequirement: {}
   InlineJavascriptRequirement: {}
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 
 # echo ${return item['path'] for item in inputs.input_files}
 inputs:

diff --git a/cwl/concat_with_comments.cwl b/cwl/concat_with_comments.cwl
@@ -8,7 +8,7 @@ baseCommand: [ 'concat_with_comments.sh' ]
 
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 
 inputs:
   comment_label:

diff --git a/cwl/cp.cwl b/cwl/cp.cwl
@@ -5,7 +5,7 @@ class: CommandLineTool
 baseCommand: cp
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 inputs:
   input_file:
     type: File

diff --git a/cwl/cut.cwl b/cwl/cut.cwl
@@ -5,7 +5,7 @@ class: CommandLineTool
 baseCommand: [ cut ]
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 stdout: output.txt
 inputs:
   field_indexes:

diff --git a/cwl/deduplicate-maf.cwl b/cwl/deduplicate-maf.cwl
@@ -29,7 +29,7 @@ NOTE: actually this order might be wrong but all that matters is that Chromosome
 requirements:
   InlineJavascriptRequirement: {}
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
   InitialWorkDirRequirement:
     listing:
       - entryname: run.sh

diff --git a/cwl/env_container.cwl b/cwl/env_container.cwl
@@ -7,7 +7,7 @@ doc: "CWL to save a copy of the execution environment for debugging"
 
 requirements:
   DockerRequirement:
-    dockerPull: mskcc/helix_filters_01:21.3.4
+    dockerPull: mskcc/helix_filters_01:21.4.1
 
 stdout: env.container.txt