Merge pull request #478 from pepkit/dev_add_local_hello_looper

add local hello_looper to ensure tests do not fail on github

.gitignore

@@ -64,7 +64,7 @@ open_pipelines/
 .coverage*
 .pytest_cache
 .vscode/
-.looper.yaml
+/tests/data/hello_looper-dev_derive/.gitignore
 
 # Reserved files for comparison
 *RESERVE*
@@ -81,4 +81,3 @@ __pycache__/
 *ipynb_checkpoints*
 hello_looper-master*
 /pipeline/
-/tests/data/hello_looper-dev_derive/

.looper.yaml

@@ -0,0 +1,5 @@
+pep_config: example/pep/path
+output_dir: .
+pipeline_interfaces:
+  sample: []
+  project: []

tests/data/hello_looper-dev_derive/.looper.yaml

@@ -0,0 +1,4 @@
+pep_config: ./project/project_config.yaml # pephub registry path or local path
+output_dir: "./results"
+pipeline_interfaces:
+  sample:  ../pipeline/pipeline_interface.yaml

tests/data/hello_looper-dev_derive/README.md

@@ -0,0 +1,18 @@
+# Hello World! example for looper
+
+This repository provides minimal working examples for the [looper pipeline submission engine](http://pep.databio.org/looper).
+
+This repository contains examples
+
+1. `/basic` - A basic example pipeline and project.
+2. `/pephub` - Example of how to point looper to PEPhub.
+3. `/pipestat` - Example of a pipeline that uses pipestat for recording results.
+4. `/csv` - How to use a pipeline with a CSV sample table (no YAML config)
+
+Each example contains:
+
+1. A looper config file (`.looper.yaml`).
+2. Sample data plus metadata in PEP format (or pointer to PEPhub).
+3. A looper-compatible pipeline.
+
+Explanation and results of running the above examples can be found at [Looper: Hello World](https://pep.databio.org/looper/code/hello-world/)

tests/data/hello_looper-dev_derive/advanced/.looper.yaml

@@ -0,0 +1,10 @@
+pep_config: project/project_config.yaml
+output_dir: "results"
+pipeline_interfaces:
+  sample:
+    - ../pipeline/pipeline_interface1_sample.yaml
+    - ../pipeline/pipeline_interface2_sample.yaml
+  project:
+    - ../pipeline/pipeline_interface1_project.yaml
+    - ../pipeline/pipeline_interface2_project.yaml
+

tests/data/hello_looper-dev_derive/advanced/.looper_advanced_pipestat.yaml

@@ -0,0 +1,9 @@
+pep_config: project/project_config.yaml
+output_dir: "results"
+pipeline_interfaces:
+  sample:
+    - ../pipeline/pipestat_pipeline_interface1_sample.yaml
+    - ../pipeline/pipestat_pipeline_interface2_sample.yaml
+pipestat:
+  results_file_path: results.yaml
+  flag_file_dir: results/flags

tests/data/hello_looper-dev_derive/advanced/pipeline/output_schema.yaml

@@ -0,0 +1,27 @@
+description: Sample objects produced by test pipeline.
+properties:
+  samples:
+    type: array
+    items:
+      type: object
+      properties:
+        test_property:
+          type: string
+          description: "Test sample property"
+          path: "~/sample/{sample_name}_file.txt"
+        test_property1:
+          type: string
+          description: "Test sample property"
+          path: "~/sample/{sample_name}_file1.txt"
+  test_property:
+    type: image
+    title: "Test title"
+    description: "Test project property"
+    thumbnail_path: "~/test_{name}.png"
+    path: "~/test_{name}.pdf"
+  test_property1:
+    type: image
+    title: "Test title1"
+    description: "Test project property1"
+    thumbnail_path: "~/test_{name}.png"
+    path: "~/test_{name}1.pdf"

tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_project.yaml

@@ -0,0 +1,11 @@
+pipeline_name: PIPELINE1
+pipeline_type: project
+output_schema: output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/col_pipeline1.py"
+command_template: >
+  {pipeline.var_templates.path} --project-name {project.name}
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R

tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_sample.yaml

@@ -0,0 +1,15 @@
+pipeline_name: PIPELINE1
+pipeline_type: sample
+input_schema: https://schema.databio.org/pep/2.0.0.yaml
+output_schema: output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/pipeline1.py"
+pre_submit:
+    python_functions:
+      - looper.write_sample_yaml
+command_template: >
+  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R

tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_project.yaml

@@ -0,0 +1,13 @@
+pipeline_name: OTHER_PIPELINE2
+pipeline_type: project
+output_schema: output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/col_pipeline2.py"
+command_template: >
+  {pipeline.var_templates.path} --project-name {project.name}
+compute:
+  size_dependent_variables: resources-project.tsv
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R

tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_sample.yaml

@@ -0,0 +1,16 @@
+pipeline_name: OTHER_PIPELINE2
+pipeline_type: sample
+output_schema: output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/other_pipeline2.py"
+pre_submit:
+    python_functions:
+      - looper.write_sample_yaml
+command_template: >
+  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+compute:
+  size_dependent_variables: resources-sample.tsv
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R

tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_output_schema.yaml

@@ -0,0 +1,5 @@
+pipeline_name: example_pipestat_pipeline
+samples:
+  number_of_lines:
+    type: integer
+    description: "Number of lines in the input file."

tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml

@@ -0,0 +1,15 @@
+pipeline_name: example_pipestat_pipeline
+pipeline_type: sample
+input_schema: https://schema.databio.org/pep/2.0.0.yaml
+output_schema: pipestat_output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/pipeline1.py"
+pre_submit:
+    python_functions:
+      - looper.write_sample_yaml
+command_template: >
+  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R

tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml

@@ -0,0 +1,17 @@
+pipeline_name: example_pipestat_pipeline
+pipeline_type: sample
+input_schema: https://schema.databio.org/pep/2.0.0.yaml
+output_schema: pipestat_output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/other_pipeline2.py"
+pre_submit:
+    python_functions:
+      - looper.write_sample_yaml
+command_template: >
+  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+compute:
+  size_dependent_variables: resources-sample.tsv
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R

tests/data/hello_looper-dev_derive/advanced/pipeline/readData.R

@@ -0,0 +1,10 @@
+readData = function(project, sampleName="sample1") {
+  lapply(getOutputsBySample(project, sampleName), function(x) {
+      lapply(x, function(x1){
+          message("Reading: ", basename(x1))
+      df = read.table(x1, stringsAsFactors=F)
+      colnames(df)[1:3] = c('chr', 'start', 'end')
+          GenomicRanges::GRanges(df)
+      })
+  })
+}

tests/data/hello_looper-dev_derive/advanced/pipeline/resources-project.tsv

@@ -0,0 +1,6 @@
+max_file_size	cores	mem	time
+0.05	1	12000	00-01:00:00
+0.5	1	16000	00-01:00:00
+1	1	16000	00-01:00:00
+10	1	16000	00-01:00:00
+NaN	1	32000	00-02:00:00

tests/data/hello_looper-dev_derive/advanced/pipeline/resources-sample.tsv

@@ -0,0 +1,7 @@
+max_file_size	cores	mem	time
+0.001	1	8000	00-04:00:00
+0.05	2	12000	00-08:00:00
+0.5	4	16000	00-12:00:00
+1	8	16000	00-24:00:00
+10	16	32000	02-00:00:00
+NaN	32	32000	04-00:00:00

tests/data/hello_looper-dev_derive/advanced/project/annotation_sheet.csv

@@ -0,0 +1,4 @@
+sample_name,protocol,data_source,SRR,Sample_geo_accession,read1,read2
+sample1,PROTO1,SRA,SRR5210416,GSM2471255,SRA_1,SRA_2
+sample2,PROTO1,SRA,SRR5210450,GSM2471300,SRA_1,SRA_2
+sample3,PROTO2,SRA,SRR5210398,GSM2471249,SRA_1,SRA_2

tests/data/hello_looper-dev_derive/advanced/project/project_config.yaml

@@ -0,0 +1,12 @@
+name: looper_advanced_test
+pep_version: "2.0.0"
+sample_table: annotation_sheet.csv
+
+sample_modifiers:
+  append:
+    attr: "val"
+  derive:
+    attributes: [read1, read2]
+    sources:
+      SRA_1: "{SRR}_1.fastq.gz"
+      SRA_2: "{SRR}_2.fastq.gz"

tests/data/hello_looper-dev_derive/basic/.looper.yaml

@@ -0,0 +1,5 @@
+pep_config: project/project_config.yaml # local path to pep config
+# pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
+output_dir: "results"
+pipeline_interfaces:
+  sample: pipeline/pipeline_interface.yaml

tests/data/hello_looper-dev_derive/basic/.looper_project.yaml

@@ -0,0 +1,4 @@
+pep_config: project/project_config.yaml # local path to pep config
+output_dir: "results"
+pipeline_interfaces:
+  project: pipeline/pipeline_interface_project.yaml

tests/data/hello_looper-dev_derive/basic/data/frog_1.txt

@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!

tests/data/hello_looper-dev_derive/basic/data/frog_2.txt

@@ -0,0 +1,7 @@
+ribbit
+ribbit
+ribbit
+
+ribbit, ribbit
+ribbit, ribbit
+CROAK!

tests/data/hello_looper-dev_derive/basic/pipeline/count_lines.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+echo "Number of lines: $linecount"

tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface.yaml

@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: sample
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} {sample.file}

tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface_project.yaml

@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: project
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} "data/*.txt"

tests/data/hello_looper-dev_derive/basic/project/project_config.yaml

@@ -0,0 +1,7 @@
+pep_version: 2.0.0
+sample_table: sample_annotation.csv
+sample_modifiers:
+  derive:
+    attributes: [file]
+    sources:
+      source1: "data/{sample_name}.txt"

tests/data/hello_looper-dev_derive/basic/project/sample_annotation.csv

@@ -0,0 +1,3 @@
+sample_name,library,file,toggle
+frog_1,anySampleType,source1,1
+frog_2,anySampleType,source1,1

tests/data/hello_looper-dev_derive/csv/.looper.yaml

@@ -0,0 +1,5 @@
+pep_config: project/sample_annotation.csv # local path to CSV
+# pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
+output_dir: "results"
+pipeline_interfaces:
+  sample: pipeline/pipeline_interface.yaml

tests/data/hello_looper-dev_derive/csv/data/frog1_data.txt

@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!

tests/data/hello_looper-dev_derive/csv/data/frog2_data.txt

@@ -0,0 +1,7 @@
+ribbit
+ribbit
+ribbit
+
+ribbit, ribbit
+ribbit, ribbit
+CROAK!

tests/data/hello_looper-dev_derive/csv/pipeline/count_lines.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+echo "Number of lines: $linecount"

tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface.yaml

@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: sample
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} {sample.file}

tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface_project.yaml

@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: project
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} "data/*.txt"

tests/data/hello_looper-dev_derive/csv/project/sample_annotation.csv

@@ -0,0 +1,3 @@
+sample_name,library,file,toggle
+frog_1,anySampleType,data/frog1_data.txt,1
+frog_2,anySampleType,data/frog2_data.txt,1

tests/data/hello_looper-dev_derive/pephub/.looper.yaml

@@ -0,0 +1,4 @@
+pep_config: pepkit/hello_looper:default # pephub registry path or local path
+output_dir: results
+pipeline_interfaces:
+  sample: pipeline/pipeline_interface.yaml

tests/data/hello_looper-dev_derive/pephub/data/frog1_data.txt

@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!

tests/data/hello_looper-dev_derive/pephub/data/frog2_data.txt

@@ -0,0 +1,7 @@
+ribbit
+ribbit
+ribbit
+
+ribbit, ribbit
+ribbit, ribbit
+CROAK!

tests/data/hello_looper-dev_derive/pephub/pipeline/count_lines.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+echo "Number of lines: $linecount"

tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface.yaml

@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: sample
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} {sample.file}

tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface_project.yaml

@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: project
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} "data/*.txt"

tests/data/hello_looper-dev_derive/pipestat/.looper.yaml

@@ -0,0 +1,8 @@
+pep_config: ./project/project_config.yaml # pephub registry path or local path
+output_dir: ./results
+pipeline_interfaces:
+  sample:  ./pipeline_pipestat/pipeline_interface.yaml
+  project: ./pipeline_pipestat/pipeline_interface_project.yaml
+pipestat:
+  results_file_path: results.yaml
+  flag_file_dir: results/flags

tests/data/hello_looper-dev_derive/pipestat/.looper_pipestat_shell.yaml

@@ -0,0 +1,7 @@
+pep_config: ./project/project_config.yaml # pephub registry path or local path
+output_dir: ./results
+pipeline_interfaces:
+  sample:  ./pipeline_pipestat/pipeline_interface_shell.yaml
+pipestat:
+  results_file_path: results.yaml
+  flag_file_dir: results/flags

tests/data/hello_looper-dev_derive/pipestat/data/frog_1.txt

@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!