From 485159c0c8e41fd4e138b5f76b9fab6960136bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:41:15 +0100 Subject: [PATCH 01/12] Add scaffolding for a general report --- .tests/integration/Snakefile_module | 1 + .tests/integration/config.yaml | 3 ++ .../config/NA12878_N.general_report.json | 0 config/config.yaml | 3 ++ config/output_files.yaml | 3 ++ docs/softwares.md | 22 ++++++++++++ workflow/Snakefile | 1 + workflow/rules/general_html_report.smk | 34 +++++++++++++++++++ workflow/schemas/config.schema.yaml | 25 ++++++++++++++ workflow/schemas/resources.schema.yaml | 20 +++++++++++ workflow/schemas/rules.schema.yaml | 23 +++++++++++++ workflow/scripts/general_html_report.py | 22 ++++++++++++ .../templates/general_html_report/index.html | 14 ++++++++ 13 files changed, 171 insertions(+) create mode 100644 .tests/integration/config/NA12878_N.general_report.json create mode 100644 workflow/rules/general_html_report.smk create mode 100644 workflow/scripts/general_html_report.py create mode 100644 workflow/templates/general_html_report/index.html diff --git a/.tests/integration/Snakefile_module b/.tests/integration/Snakefile_module index 302a1d3..399a432 100644 --- a/.tests/integration/Snakefile_module +++ b/.tests/integration/Snakefile_module @@ -4,6 +4,7 @@ from hydra_genetics.utils.misc import get_module_snakefile rule all: input: "reports/cnv_html_report/NA12878_N.pathology.cnv_report.html", + "reports/general_html_report/NA12878_N.general_report.html", module reports: diff --git a/.tests/integration/config.yaml b/.tests/integration/config.yaml index 33db5d0..9e7891f 100644 --- a/.tests/integration/config.yaml +++ b/.tests/integration/config.yaml @@ -9,6 +9,9 @@ reference: cnv_html_report: cytobands: true +general_html_report: + json: "config/{sample}_{type}.general_report.json" + merge_cnv_json: annotations: - config/amp_genes.bed diff --git a/.tests/integration/config/NA12878_N.general_report.json b/.tests/integration/config/NA12878_N.general_report.json new file mode 100644 index 0000000..e69de29 diff --git a/config/config.yaml b/config/config.yaml index 656224c..da93564 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -14,6 +14,9 @@ cnv_html_report: cytobands: false show_table: true +general_html_report: + json: "reports/general_html_report/{sample}_{type}.general_report.json" + merge_cnv_json: annotations: cytobands: diff --git a/config/output_files.yaml b/config/output_files.yaml index 136b1fb..8a836e9 100644 --- a/config/output_files.yaml +++ b/config/output_files.yaml @@ -2,3 +2,6 @@ files: - name: "CNV HTML report" input: "reports/cnv_html_report/{sample}_{type}.pathology.cnv_report.html" output: "results/cnv/{sample}_{type}.pathology.cnv_report.html" + - name: "General HTML report" + input: "reports/general_html_report/{sample}_{type}.general_report.html" + output: "results/reports/{sample}_{type}.general_report.html" diff --git a/docs/softwares.md b/docs/softwares.md index 6ce2413..1caf123 100644 --- a/docs/softwares.md +++ b/docs/softwares.md @@ -65,3 +65,25 @@ Merge JSON files from multiple CNV callers and add annotations and other sample #### Resources settings (`resources.yaml`) #RESOURCESSCHEMA__merge_cnv_json# + +## general_html_report + +Generate a general HTML report for a single sample. + +### :snake: Rule + +#SNAKEMAKE_RULE_SOURCE__general_html_report__general_html_report# + +#### :left_right_arrow: input / output files + +#SNAKEMAKE_RULE_TABLE__general_html_report__general_html_report# + +### :wrench: Configuration + +#### Software settings (`config.yaml`) + +#CONFIGSCHEMA__general_html_report# + +#### Resources settings (`resources.yaml`) + +#RESOURCESSCHEMA__general_html_report# diff --git a/workflow/Snakefile b/workflow/Snakefile index f0d23b0..34eaf3f 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -6,6 +6,7 @@ __license__ = "GPL-3" # Include pipeline specific rules include: "rules/common.smk" +include: "rules/general_html_report.smk" include: "rules/cnv_html_report.smk" diff --git a/workflow/rules/general_html_report.smk b/workflow/rules/general_html_report.smk new file mode 100644 index 0000000..39184c4 --- /dev/null +++ b/workflow/rules/general_html_report.smk @@ -0,0 +1,34 @@ +__author__ = "Niklas Mähler" +__copyright__ = "Copyright 2023, Niklas Mähler" +__email__ = "niklas.mahler@regionvasterbotten.se" +__license__ = "GPL-3" + + +rule general_html_report: + input: + json=config.get("general_html_report", {}).get("json"), + html_template=workflow.source_path("../templates/general_html_report/index.html"), + output: + html="reports/general_html_report/{sample}_{type}.general_report.html", + params: + extra=config.get("general_html_report", {}).get("extra", ""), + log: + "reports/general_html_report/{sample}_{type}.general_report.log", + benchmark: + repeat( + "reports/general_html_report/{sample}_{type}.output.benchmark.tsv", + config.get("general_html_report", {}).get("benchmark_repeats", 1), + ) + threads: config.get("general_html_report", {}).get("threads", config["default_resources"]["threads"]) + resources: + mem_mb=config.get("general_html_report", {}).get("mem_mb", config["default_resources"]["mem_mb"]), + mem_per_cpu=config.get("general_html_report", {}).get("mem_per_cpu", config["default_resources"]["mem_per_cpu"]), + partition=config.get("general_html_report", {}).get("partition", config["default_resources"]["partition"]), + threads=config.get("general_html_report", {}).get("threads", config["default_resources"]["threads"]), + time=config.get("general_html_report", {}).get("time", config["default_resources"]["time"]), + container: + config.get("general_html_report", {}).get("container", config["default_container"]) + message: + "{rule}: generate general html report from json config {input.json}" + script: + "../scripts/general_html_report.py" diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml index 677d0ba..06d95ce 100644 --- a/workflow/schemas/config.schema.yaml +++ b/workflow/schemas/config.schema.yaml @@ -62,6 +62,31 @@ properties: type: string description: name or path to docker/singularity container + general_html_report: + type: object + description: parameters for general_html_report + properties: + json: + type: string + format: uri-reference + description: | + Path to the sample-specific configuration of the report. The wildcards + `sample` and `type` are supported. + examples: + - "report_configs/{sample}_{type}/general_report.json" + - "report_configs/general_html_report/{sample}_{type}.general_report.json" + benchmark_repeats: + type: integer + description: set number of times benchmark should be repeated + container: + type: string + description: name or path to docker/singularity container + extra: + type: string + description: parameters that should be forwarded + required: + - json + merge_cnv_json: type: object description: parameters for merge_cnv_json diff --git a/workflow/schemas/resources.schema.yaml b/workflow/schemas/resources.schema.yaml index 842e6d7..8ecdf7f 100644 --- a/workflow/schemas/resources.schema.yaml +++ b/workflow/schemas/resources.schema.yaml @@ -67,6 +67,26 @@ properties: type: string description: max execution time + general_html_report: + type: object + description: resource definitions for general_html_report + properties: + mem_mb: + type: integer + description: max memory in MB to be available + mem_per_cpu: + type: integer + description: memory in MB used per cpu + partition: + type: string + description: partition to use on cluster + threads: + type: integer + description: number of threads to be available + time: + type: string + description: max execution time + merge_cnv_json: type: object description: resource definitions for merge_cnv_json diff --git a/workflow/schemas/rules.schema.yaml b/workflow/schemas/rules.schema.yaml index e5fe303..b816a02 100644 --- a/workflow/schemas/rules.schema.yaml +++ b/workflow/schemas/rules.schema.yaml @@ -83,6 +83,29 @@ properties: description: > A JSON representation of the CNV results from a specific caller. + general_html_report: + type: object + description: input and output parameters for general_html_report + properties: + input: + type: object + description: list of inputs + properties: + json: + type: string + description: sample-specific configuration of the report + html_template: + type: string + description: path to the html template to use for the report + + output: + type: object + description: list of outputs + properties: + html: + type: string + description: path to the generated report + merge_cnv_json: type: object description: input and output parameters for merge_cnv_json diff --git a/workflow/scripts/general_html_report.py b/workflow/scripts/general_html_report.py new file mode 100644 index 0000000..57ed703 --- /dev/null +++ b/workflow/scripts/general_html_report.py @@ -0,0 +1,22 @@ +from jinja2 import Template + + +def generate_report(template_filename, json): + with open(template_filename) as f: + template = Template(source=f.read()) + + return template.render(dict(sample=snakemake.wildcards.sample)) + + +def main(): + html_template = snakemake.input.html_template + json = snakemake.input.json + + report_content = generate_report(html_template, json) + + with open(snakemake.output.html, "w") as f: + f.write(report_content) + + +if __name__ == "__main__": + main() diff --git a/workflow/templates/general_html_report/index.html b/workflow/templates/general_html_report/index.html new file mode 100644 index 0000000..be34175 --- /dev/null +++ b/workflow/templates/general_html_report/index.html @@ -0,0 +1,14 @@ + + + + {{ sample }} – General Report + + + + + +
+

{{ sample }}

+
+ + From 021b0c6a06ba7b284690cdbe9c287a8d6d37d3b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:14:05 +0100 Subject: [PATCH 02/12] Parse and validate report config --- .../config/NA12878_N.general_report.json | 16 ++++++ workflow/rules/general_html_report.smk | 3 +- .../general_html_report_json.schema.yaml | 49 +++++++++++++++++++ workflow/scripts/general_html_report.py | 32 ++++++++++-- .../templates/general_html_report/index.html | 32 +++++++++++- 5 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 workflow/schemas/general_html_report_json.schema.yaml diff --git a/.tests/integration/config/NA12878_N.general_report.json b/.tests/integration/config/NA12878_N.general_report.json index e69de29..7ee8ce9 100644 --- a/.tests/integration/config/NA12878_N.general_report.json +++ b/.tests/integration/config/NA12878_N.general_report.json @@ -0,0 +1,16 @@ +{ + "sample": "NA12878", + "analysis_date": "2023-11-17", + "pipeline": { + "name": "Twist Solid", + "version": "0.9.0", + "uri": "https://github.com/genomic-medicine-sweden/Twist_Solid/tree/v0.9.0" + }, + "file_links": [ + { + "name": "CNV HTML report", + "description": "HTML report summarising the results of the CNV analysis", + "uri": "results/cnv/NA12878_N.pathology.cnv_report.html" + } + ] +} diff --git a/workflow/rules/general_html_report.smk b/workflow/rules/general_html_report.smk index 39184c4..1d3b88b 100644 --- a/workflow/rules/general_html_report.smk +++ b/workflow/rules/general_html_report.smk @@ -6,8 +6,9 @@ __license__ = "GPL-3" rule general_html_report: input: - json=config.get("general_html_report", {}).get("json"), + config_schema=workflow.source_path("../schemas/general_html_report_json.schema.yaml"), html_template=workflow.source_path("../templates/general_html_report/index.html"), + json=config.get("general_html_report", {}).get("json"), output: html="reports/general_html_report/{sample}_{type}.general_report.html", params: diff --git a/workflow/schemas/general_html_report_json.schema.yaml b/workflow/schemas/general_html_report_json.schema.yaml new file mode 100644 index 0000000..4f0b75e --- /dev/null +++ b/workflow/schemas/general_html_report_json.schema.yaml @@ -0,0 +1,49 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: General Report JSON Config +description: Configuration of a general HTML report for a Hydra Genetics pipeline +type: object +properties: + sample: + type: string + description: Name of the sample + + analysis_date: + type: string + description: Date of the analysis + + pipeline: + type: object + description: Pipeline information + properties: + name: + type: string + description: Name of the pipeline + version: + type: string + description: Version of the pipeline + uri: + type: string + format: uri-reference + description: URI of the pipeline + + file_links: + type: array + description: List of file links + items: + type: object + properties: + name: + type: string + description: Descriptive name of the file + description: + type: string + description: Description of the file + uri: + type: string + format: uri-reference + description: URI of the file + +required: + - sample + - analysis_date + - pipeline diff --git a/workflow/scripts/general_html_report.py b/workflow/scripts/general_html_report.py index 57ed703..a02e5e4 100644 --- a/workflow/scripts/general_html_report.py +++ b/workflow/scripts/general_html_report.py @@ -1,18 +1,42 @@ from jinja2 import Template +import json +from jsonschema import validate +import time +import yaml -def generate_report(template_filename, json): +def validate_dict(d: dict, schema_path: str): + with open(schema_path) as f: + validate(instance=d, schema=yaml.safe_load(f)) + + +def generate_report(template_filename: str, config: dict): with open(template_filename) as f: template = Template(source=f.read()) - return template.render(dict(sample=snakemake.wildcards.sample)) + return template.render( + dict( + metadata=dict( + analysis_date=config["analysis_date"], + report_date=time.strftime("%Y-%m-%d %H:%M", time.localtime()), + sample=config["sample"], + ), + pipeline=config["pipeline"], + file_links=config["file_links"], + ) + ) def main(): html_template = snakemake.input.html_template - json = snakemake.input.json + json_file = snakemake.input.json + + with open(json_file) as f: + config = json.load(f) + + validate_dict(config, snakemake.input.config_schema) - report_content = generate_report(html_template, json) + report_content = generate_report(html_template, config) with open(snakemake.output.html, "w") as f: f.write(report_content) diff --git a/workflow/templates/general_html_report/index.html b/workflow/templates/general_html_report/index.html index be34175..f4680cc 100644 --- a/workflow/templates/general_html_report/index.html +++ b/workflow/templates/general_html_report/index.html @@ -1,14 +1,42 @@ - {{ sample }} – General Report + {{ metadata.sample }} – General Report
-

{{ sample }}

+

{{ metadata.sample }}

+

Report generated at {{ metadata.report_date }}

+

Sample analysed at {{ metadata.analysis_date }}

+ +

+ + hydra-genetics/reports v0.2.0 + +

+
+ +
+

Pipeline

+ +
+ + {% for fl in file_links %} +
+

{{ fl.name }}

+

{{ fl.description }}

+

{{ fl.uri }}

+
+ {% endfor %} From e5b3251616409197a57e37f6fe4ae0d513064d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:59:36 +0100 Subject: [PATCH 03/12] Add parameter for final directory depth of the report This is something that is needed in order to correctly resolve relative paths in the config. It is possible that this could be handled in a more automated way, for example by looking at the output files of the pipeline. This works as a start. --- .tests/integration/config.yaml | 1 + docs/{reports.md => cnv_report.md} | 0 workflow/rules/general_html_report.smk | 1 + workflow/schemas/config.schema.yaml | 8 ++++++++ workflow/scripts/general_html_report.py | 9 +++++++-- 5 files changed, 17 insertions(+), 2 deletions(-) rename docs/{reports.md => cnv_report.md} (100%) diff --git a/.tests/integration/config.yaml b/.tests/integration/config.yaml index 9e7891f..b138112 100644 --- a/.tests/integration/config.yaml +++ b/.tests/integration/config.yaml @@ -11,6 +11,7 @@ cnv_html_report: general_html_report: json: "config/{sample}_{type}.general_report.json" + final_directory_depth: 2 merge_cnv_json: annotations: diff --git a/docs/reports.md b/docs/cnv_report.md similarity index 100% rename from docs/reports.md rename to docs/cnv_report.md diff --git a/workflow/rules/general_html_report.smk b/workflow/rules/general_html_report.smk index 1d3b88b..135ab1a 100644 --- a/workflow/rules/general_html_report.smk +++ b/workflow/rules/general_html_report.smk @@ -12,6 +12,7 @@ rule general_html_report: output: html="reports/general_html_report/{sample}_{type}.general_report.html", params: + final_directory_depth=config.get("general_html_report", {}).get("final_directory_depth", 1), extra=config.get("general_html_report", {}).get("extra", ""), log: "reports/general_html_report/{sample}_{type}.general_report.log", diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml index 06d95ce..2426efc 100644 --- a/workflow/schemas/config.schema.yaml +++ b/workflow/schemas/config.schema.yaml @@ -75,6 +75,13 @@ properties: examples: - "report_configs/{sample}_{type}/general_report.json" - "report_configs/general_html_report/{sample}_{type}.general_report.json" + final_directory_depth: + type: integer + description: | + How deep in the final results directory the report will be. This + will be used to correctly resolve relative paths in the JSON config. + For example, if the report is located in the directory `results/reports`, + the depth would be 2. benchmark_repeats: type: integer description: set number of times benchmark should be repeated @@ -86,6 +93,7 @@ properties: description: parameters that should be forwarded required: - json + - final_directory_depth merge_cnv_json: type: object diff --git a/workflow/scripts/general_html_report.py b/workflow/scripts/general_html_report.py index a02e5e4..ee9915b 100644 --- a/workflow/scripts/general_html_report.py +++ b/workflow/scripts/general_html_report.py @@ -10,10 +10,14 @@ def validate_dict(d: dict, schema_path: str): validate(instance=d, schema=yaml.safe_load(f)) -def generate_report(template_filename: str, config: dict): +def generate_report(template_filename: str, config: dict, final_directory_depth: int): with open(template_filename) as f: template = Template(source=f.read()) + if final_directory_depth != 0: + for d in config["file_links"]: + d["uri"] = final_directory_depth * "../" + d["uri"] + return template.render( dict( metadata=dict( @@ -30,13 +34,14 @@ def generate_report(template_filename: str, config: dict): def main(): html_template = snakemake.input.html_template json_file = snakemake.input.json + final_directory_depth = snakemake.params.final_directory_depth with open(json_file) as f: config = json.load(f) validate_dict(config, snakemake.input.config_schema) - report_content = generate_report(html_template, config) + report_content = generate_report(html_template, config, final_directory_depth) with open(snakemake.output.html, "w") as f: f.write(report_content) From e7bdbc444f68ee38b5225a74b0b56801369aff30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Mon, 4 Dec 2023 14:01:59 +0100 Subject: [PATCH 04/12] Urlencode URLs and trim `../` from the file links --- workflow/templates/general_html_report/index.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/workflow/templates/general_html_report/index.html b/workflow/templates/general_html_report/index.html index f4680cc..1e21cb2 100644 --- a/workflow/templates/general_html_report/index.html +++ b/workflow/templates/general_html_report/index.html @@ -27,7 +27,10 @@

Pipeline

@@ -35,7 +38,7 @@

Pipeline

{{ fl.name }}

{{ fl.description }}

-

{{ fl.uri }}

+

{{ fl.uri.lstrip("./") }}

{% endfor %} From 2fc0b0c1b910f8315cd346527f1ba11001d279d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Mon, 4 Dec 2023 16:20:55 +0100 Subject: [PATCH 05/12] Add file links, single_value results and table results This is a start. All the other types should be implemented, and proper unit tests should be added for the script generating the report. As a consequence I might have to make things a bit more granular in order for things to be easily testable. --- .../config/NA12878_N.general_report.json | 27 ++++++++++ .../general_html_report_json.schema.yaml | 46 +++++++++++++++++ workflow/scripts/general_html_report.py | 1 + .../templates/general_html_report/index.html | 49 ++++++++++++++++--- 4 files changed, 117 insertions(+), 6 deletions(-) diff --git a/.tests/integration/config/NA12878_N.general_report.json b/.tests/integration/config/NA12878_N.general_report.json index 7ee8ce9..d770f9a 100644 --- a/.tests/integration/config/NA12878_N.general_report.json +++ b/.tests/integration/config/NA12878_N.general_report.json @@ -12,5 +12,32 @@ "description": "HTML report summarising the results of the CNV analysis", "uri": "results/cnv/NA12878_N.pathology.cnv_report.html" } + ], + "results": [ + { + "name": "TMB", + "type": "single_value", + "description": "Tumor mutational burden", + "value": 12 + }, + { + "name": "Result 2", + "type": "table", + "description": "A table of results", + "value": [ + { + "column1": "row1", + "column2": "value1" + }, + { + "column1": "row2", + "column2": "value2" + }, + { + "column2": "value3", + "column1": "row3" + } + ] + } ] } diff --git a/workflow/schemas/general_html_report_json.schema.yaml b/workflow/schemas/general_html_report_json.schema.yaml index 4f0b75e..fa32d37 100644 --- a/workflow/schemas/general_html_report_json.schema.yaml +++ b/workflow/schemas/general_html_report_json.schema.yaml @@ -29,6 +29,7 @@ properties: file_links: type: array description: List of file links + default: [] items: type: object properties: @@ -42,8 +43,53 @@ properties: type: string format: uri-reference description: URI of the file + required: + - name + - description + - uri + + results: + type: array + description: List of results to present + default: [] + items: + type: object + oneOf: + - type: object + properties: + name: + type: string + description: A descriptive name of the result + description: + type: string + description: A more detailed description of the result + type: + type: string + description: The type of result + enum: + - file_table + - image + - plain_text + - single_value + - table + value: + description: | + The value of the result. Exactly what this is depends on the type. + + - file_table: a delimited text file (csv or tsv) with a header. + - image: the path to a png image. + - plain_text: a string containing the text to be presented. + - single_value: a single value, e.g. a number. + - table: a json representation of a table. + required: + - name + - description + - type + - value required: - sample - analysis_date - pipeline + - file_links + - results diff --git a/workflow/scripts/general_html_report.py b/workflow/scripts/general_html_report.py index ee9915b..879beed 100644 --- a/workflow/scripts/general_html_report.py +++ b/workflow/scripts/general_html_report.py @@ -27,6 +27,7 @@ def generate_report(template_filename: str, config: dict, final_directory_depth: ), pipeline=config["pipeline"], file_links=config["file_links"], + results=config["results"], ) ) diff --git a/workflow/templates/general_html_report/index.html b/workflow/templates/general_html_report/index.html index 1e21cb2..9571650 100644 --- a/workflow/templates/general_html_report/index.html +++ b/workflow/templates/general_html_report/index.html @@ -29,17 +29,54 @@

Pipeline

  • Version: {{ pipeline.version }}
  • Source: - {{ pipeline.uri }} + {{ pipeline.uri }}
    • - {% for fl in file_links %}
    -

    {{ fl.name }}

    -

    {{ fl.description }}

    -

    {{ fl.uri.lstrip("./") }}

    +

    File links

    + + {% for fl in file_links %} +
    +

    {{ fl.name }}

    +

    {{ fl.description }}

    +

    {{ fl.uri.lstrip("./") }}

    +
    + {% endfor %} +
    + +
    +

    Results

    + + {% for r in results %} +
    +

    {{ r.name }}

    +

    {{ r.description }}

    + {% if r.type == "table" %} + + + + {% for c in r.value[0].keys() %} + + {% endfor %} + + + + {% for row in r.value %} + + {% for k in r.value[0].keys() %} + + {% endfor %} + + {% endfor %} + +
    {{ c }}
    {{ row[k] }}
    + {% elif r.type == "single_value" %} +

    {{ r.value }}

    + {% endif %} +
    + {% endfor %}
    - {% endfor %} From 486244864fae89d7d1add39e85f873852e96a140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Mon, 4 Dec 2023 16:30:44 +0100 Subject: [PATCH 06/12] Add image support One thing I have to do here is to check the URL and if it is a path to a local file, add the relative redirect needed in order to resolve the path properly. --- .tests/integration/config/NA12878_N.general_report.json | 6 ++++++ workflow/templates/general_html_report/index.html | 2 ++ 2 files changed, 8 insertions(+) diff --git a/.tests/integration/config/NA12878_N.general_report.json b/.tests/integration/config/NA12878_N.general_report.json index d770f9a..7162058 100644 --- a/.tests/integration/config/NA12878_N.general_report.json +++ b/.tests/integration/config/NA12878_N.general_report.json @@ -38,6 +38,12 @@ "column1": "row3" } ] + }, + { + "name": "Image result", + "type": "image", + "description": "An image", + "value": "https://placehold.co/600x400/png" } ] } diff --git a/workflow/templates/general_html_report/index.html b/workflow/templates/general_html_report/index.html index 9571650..3ebf70f 100644 --- a/workflow/templates/general_html_report/index.html +++ b/workflow/templates/general_html_report/index.html @@ -74,6 +74,8 @@

    {{ r.name }}

    {% elif r.type == "single_value" %}

    {{ r.value }}

    + {% elif r.type == "image" %} + {% endif %} {% endfor %} From f1065fe28b0750116c53a566cae7352c400369ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:34:45 +0100 Subject: [PATCH 07/12] Validate table structure This also adds a unit test for the table validation function. --- .tests/unit/test_general_html_report.py | 73 +++++++++++++++++++++++++ workflow/scripts/general_html_report.py | 25 ++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 .tests/unit/test_general_html_report.py diff --git a/.tests/unit/test_general_html_report.py b/.tests/unit/test_general_html_report.py new file mode 100644 index 0000000..b09b7df --- /dev/null +++ b/.tests/unit/test_general_html_report.py @@ -0,0 +1,73 @@ +import pathlib +import pytest +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parents[2])) + +from workflow.scripts.general_html_report import * + + +def test_valid_table(): + table = [ + { + "col1": "val1", + "col2": 1, + }, + { + "col1": "val2", + "col2": 2, + }, + { + "col1": "val3", + "col2": 3, + }, + ] + + assert validate_table_data(table) + + +def test_empty_table(): + with pytest.raises(ValueError, match="empty table"): + validate_table_data([]) + + +def test_unequal_columns(): + table = [ + { + "col1": "val1", + "col2": 1, + }, + { + "col1": "val2", + "col3": 2, + }, + { + "col1": "val3", + "col2": 3, + }, + ] + + with pytest.raises( + ValueError, + match="expected columns 'col1', 'col2' in row 2, found 'col1', 'col3'", + ): + validate_table_data(table) + + +def test_missing_columns(): + table = [ + { + "col1": "val1", + "col2": 1, + }, + { + "col1": "val2", + }, + { + "col1": "val3", + "col2": 3, + }, + ] + + with pytest.raises(ValueError, match="expected 2 columns in row 2, found 1"): + validate_table_data(table) diff --git a/workflow/scripts/general_html_report.py b/workflow/scripts/general_html_report.py index 879beed..ad6237d 100644 --- a/workflow/scripts/general_html_report.py +++ b/workflow/scripts/general_html_report.py @@ -10,7 +10,26 @@ def validate_dict(d: dict, schema_path: str): validate(instance=d, schema=yaml.safe_load(f)) -def generate_report(template_filename: str, config: dict, final_directory_depth: int): +def validate_table_data(table: list) -> bool: + if len(table) == 0: + raise ValueError("empty table") + + n_cols = len(table[0]) + cols = table[0].keys() + + for i, row in enumerate(table, start=1): + if len(row) != n_cols: + raise ValueError(f"expected {n_cols} columns in row {i}, found {len(row)}") + if set(row.keys()) != set(cols): + raise ValueError( + f"expected columns {', '.join(repr(x) for x in cols)} " + f"in row {i}, found {', '.join(repr(x) for x in row.keys())}" + ) + + return True + + +def generate_report(template_filename: str, config: dict, final_directory_depth: int) -> str: with open(template_filename) as f: template = Template(source=f.read()) @@ -18,6 +37,10 @@ def generate_report(template_filename: str, config: dict, final_directory_depth: for d in config["file_links"]: d["uri"] = final_directory_depth * "../" + d["uri"] + for d in config["results"]: + if d["type"] == "table": + validate_table_data(d["value"]) + return template.render( dict( metadata=dict( From d3c1eedf96dd49e33e782ed8a1b9afff03823c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:52:42 +0100 Subject: [PATCH 08/12] Add support for adding local images --- .../config/NA12878_N.general_report.json | 6 ++++++ .tests/integration/test_data/NA12878_N.png | Bin 0 -> 3865 bytes .tests/unit/test_general_html_report.py | 8 ++++++++ workflow/scripts/general_html_report.py | 15 ++++++++++++--- 4 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 .tests/integration/test_data/NA12878_N.png diff --git a/.tests/integration/config/NA12878_N.general_report.json b/.tests/integration/config/NA12878_N.general_report.json index 7162058..b202764 100644 --- a/.tests/integration/config/NA12878_N.general_report.json +++ b/.tests/integration/config/NA12878_N.general_report.json @@ -44,6 +44,12 @@ "type": "image", "description": "An image", "value": "https://placehold.co/600x400/png" + }, + { + "name": "Local image", + "type": "image", + "description": "A local image from a path relative to the analysis directory", + "value": "test_data/NA12878_N.png" } ] } diff --git a/.tests/integration/test_data/NA12878_N.png b/.tests/integration/test_data/NA12878_N.png new file mode 100644 index 0000000000000000000000000000000000000000..ce3274e41656c52fbc431faac660f82dd972d95e GIT binary patch literal 3865 zcmeH~=Tj5t7RCd3QBW)(f>^rGNOsG)@*2mxad5C|+HvCw-{kS0h8 zNEJd6;s%itO$aSO6eQFnQbMT7&AlJ)UvNL%x#Neo%rozM=FBs{b7synyBp?WqCimq z03c>*VR8!q5cq?C|9C{0zu%%wIq(mWdlt?p0N|*^zm@dxSi%TG9R14sxp`K1z;n6o-Y@1GpJl z7d``oJ+DMwiQ|o;0f&I^0mqKu1qBlU!ngB;Z##*kUwI|)A0_Lj^3#ofv+@5!0#9m| zSY+HdR_8%vPPJ*`!q;X4Q7DvLjv6$7x;rLji+iw_^*EZtoL=2xP&d{Lnej)ZeEH9d zj3T!Jw^qjLvIyMGCGzS_XJn&_BX+P49LAhlU8YnwG8l}B)?nJ`@rmx|y1_Gp5B#)( z*2DObCaA5!j=;T*1tRMB8SO1&(5YJ*pb$35rYK;1lYl3=A7ns*XH^Rb-B^B?cud2C*()5C`@jX*TgS| zrDCx&7_zXV26w1H*E|V5`{9}QVv*rPjU&6>=L|ypnkJa>RpW~L(n+#~=d1`}?3Mnq zkg2%CCx)v#(%&Mk4EIYjTUuMkusiLr1`K@siSIBt+`Th`0^-)Xwq^sVS-q+LJ0?IF zBtdbc%9?6}+*zM5*9r|+8q?iMkh+vj+I}A^6gP0wkpLUKt1c&XBfFX4Bev2Lclg(n z?!a~pjn9S8aLVm_gt9%C46Q%#rv?0_OEY%n2XcPhw9lGnQBO}68S!mC^7dOI zVT+cSJgw;2l-ZQlY`mi2xe2n)F>%H#qDN4hBJ%#La2KC-sDVjV0cw{YV7C0F$|~tu zL$z`vve4kcavd=^C0%WD(y1{a-Dj|VM>Az}86@ZE%3zHjn=EQnYF0eSi}WCaswxrJjAOX_)^!)mllR%q znq=*k6h#ecSQh(&H7p%7k2s0|U?M=LGTp_SjHTFrWAe*|K~;|@&AuZ}F^F!sP{a>T zAs8JoVtGp(tA_4PY15!v^${ADB?tW07ppAx24~FdyaTcCFvYBM! zFjn$=Cn2}Y>me_rfI4Ked~oSH4Otp;1X%R+%j#62f9VhPW2(+2aJzo)c@C;XVYy*< z=R-h*Zn>k5R1Tawto#}dH8d-2j*R91J3a~2oAmvLq!uYnh<+;^{Y+AAFY%9a!*uew z!`Ajf@7R#)&+OSM_>x}SL-=*1-SZP>%jJ_gO&Bb-!lyd%@M3mD$pMrD&HgeN`DBvv zHPzqLs7yTLtSB#3{$~H33kQ>u5TCnTn|{;h*fpsQ35KM~w1=dsg0XG=4%ip!cV)(3IA{#r6GOXY7kZ>!XI_W)TkhaIVYxu$oyqH`oc`4B5q4o9mHiIah<2HL!*xLKa4Rha?-sT^lYQ$wzU zW6-2E({E{&mBme}Oo4~)^EyIOu!h;mvWyiR1*yzkc)>sj5A<>pu3@>qt$brZ zR}%PgvBwv6S1U4d6guw%goVRwRT7*{Wwa5M_KndFN$2q!kW6f64D6WV?0tThQq9L| zUak?meExGub;6(8HDIz!g|lhc@7vNi>>xuMZ_p~_=)!UFbdn_+t&@miW7xi76Hs-< zyWh^WAHWACeh<}%GE9rzVl%cT(|?ej9u;gpgmL&Kz52ysdUKA@+j>#0~h6G zZ-P7+;{mpu0EsC^vR9Ix2u97C)r=6kp7POov?h>QzSk6L5S0S;f#{SLvzYW2VfnF> z#^|0FVJ|1xO-!YrOO#cwJ5>*8^v%nm@ooeJrjx@gb2ja6T)L38UpPIb2Sn6*&Q+St z=ZXeAcKR~+REm{P8*)W;uvGB1`U<|ym3`eBwQ7yP#?^ z92Ek_tt{KZqebGw=2#!Bf1j#z^*y}jO$@FW`ozZ;Mk}mu@wJ~H;fRYCAWD&Apnv3` zMdDlSv@R9jQ+p^9f4QbB+RlIbgLDM%T;0y(%YK5{Nr(CYGsor$_H0`5S8b0;iZ?7> z5<+Y2dD7eHzQoYM?VP7z%YA6bu^+RKDi60=ZBDCxt-aV`;qbd#6w? zZ1Ik2nM>EjT5Jh0oc!!tNA!I5O=Wr9H_kXXnzeFYK1fzqj+anC4pX-BP^Ilpn bool: return True +def fix_relative_uri(uri: str, depth: int) -> str: + if re.match("^([^/]+:|/)", uri): + return uri + + return depth * "../" + uri + + def generate_report(template_filename: str, config: dict, final_directory_depth: int) -> str: with open(template_filename) as f: template = Template(source=f.read()) - if final_directory_depth != 0: - for d in config["file_links"]: - d["uri"] = final_directory_depth * "../" + d["uri"] + for d in config["file_links"]: + d["uri"] = fix_relative_uri(d["uri"], final_directory_depth) for d in config["results"]: if d["type"] == "table": validate_table_data(d["value"]) + if d["type"] == "image": + d["value"] = fix_relative_uri(d["value"], final_directory_depth) return template.render( dict( From c826268e94eb4a063e135a0162de6be7e72c3b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:54:49 +0100 Subject: [PATCH 09/12] Add general report template to release-please files --- .github/workflows/release-please.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 345ad41..b6d589a 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -16,3 +16,4 @@ jobs: default-branch: main extra-files: - workflow/templates/cnv_html_report/index.html + - workflow/templates/general_html_report/index.html From 98569e31eb86b8727b782d8a4ae7705d35166751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Fri, 8 Dec 2023 16:46:05 +0100 Subject: [PATCH 10/12] Add tailwindcss --- .gitignore | 2 + .prettierignore | 1 + docs/tailwind.md | 30 + mkdocs.yaml | 2 + tailwind.config.js | 8 + workflow/templates/assets/css/main.css | 631 +++++++++++++++++++++ workflow/templates/assets/css/tailwind.css | 9 + 7 files changed, 683 insertions(+) create mode 100644 .prettierignore create mode 100644 docs/tailwind.md create mode 100644 tailwind.config.js create mode 100644 workflow/templates/assets/css/main.css create mode 100644 workflow/templates/assets/css/tailwind.css diff --git a/.gitignore b/.gitignore index ea36e68..b486f01 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ reports/ site/ logs/ results/ +tailwindcss +*.min.css diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..c791f9d --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +workflow/templates/assets/css/main.css diff --git a/docs/tailwind.md b/docs/tailwind.md new file mode 100644 index 0000000..1c55c7e --- /dev/null +++ b/docs/tailwind.md @@ -0,0 +1,30 @@ +# Tailwind CSS + +All reports in this repository share a single CSS file by default, and the content of this file is managed by [Tailwind](https://tailwindcss.com). This is a CSS framework that works with a very large number of minimal classes, which makes the styling very flexible without you having to write a single line of CSS. Since only the classes that are actually being used are included, the size of the resulting file can be kept down. + +## Setup + +Tailwind is available through [npm](https://www.npmjs.com/package/tailwindcss), but since there are no other Node.js dependencies in this project, it is recommended that you install the [standalone CLI tool](https://tailwindcss.com/blog/standalone-cli) instead. The binaries for various systems are [hosted on Github](https://github.com/tailwindlabs/tailwindcss/releases). Here is how to install on Linux: + +```bash +curl -sLO https://github.com/tailwindlabs/tailwindcss/releases/download/v3.3.6/tailwindcss-linux-x64 +chmod +x tailwindcss-linux-x64 +mv tailwindcss-linux-x64 tailwindcss +``` + +Preferrably move this binary to a directory in your `$PATH`. Depending on the platform, the installation procedure might look different. + +## Usage + +The Tailwind configuration can be found in `tailwind.config.js`. By default, it will look for classes in all HTML and JS files in the `workflow/templates` directory. When actively working on a template, it is recommended that Tailwind is set to watch files for changes and edit the CSS file as it goes: + +```bash +tailwindcss \ + -i ./workflow/templates/assets/css/tailwind.css \ + -o ./workflow/templates/assets/css/main.css \ + --watch +``` + +In the example above, `./workflow/templates/assets/css/tailwind.css` is the input css file, and `./workflow/templates/assets/css/main.css` is the output css file which is used in the each template. This approach will make certain that each and every Tailwind class in use is available in the generated CSS file, and at the same time remove any unused classes. + +When including this file in a template, it is recommended that it is minified. In order to keep the git history clean, the minified CSS should not be committed to the repo, but rather be something that is generated by the script of each individual report. diff --git a/mkdocs.yaml b/mkdocs.yaml index 455e29b..65d5907 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -3,6 +3,8 @@ nav: - Introduction: intro.md - Reports: reports.md - Softwares: softwares.md + - Development: + - Tailwind CSS: tailwind.md theme: readthedocs extra_css: [extra.css] diff --git a/tailwind.config.js b/tailwind.config.js new file mode 100644 index 0000000..36784bb --- /dev/null +++ b/tailwind.config.js @@ -0,0 +1,8 @@ +/** @type {import('tailwindcss').Config} */ +module.exports = { + content: ["workflow/templates/**/*.{html,js}"], + theme: { + extend: {}, + }, + plugins: [], +}; diff --git a/workflow/templates/assets/css/main.css b/workflow/templates/assets/css/main.css new file mode 100644 index 0000000..e7a4815 --- /dev/null +++ b/workflow/templates/assets/css/main.css @@ -0,0 +1,631 @@ +/* +! tailwindcss v3.3.6 | MIT License | https://tailwindcss.com +*/ + +/* +1. Prevent padding and border from affecting element width. (https://github.com/mozdevs/cssremedy/issues/4) +2. Allow adding a border to an element by just adding a border-width. (https://github.com/tailwindcss/tailwindcss/pull/116) +*/ + +*, +::before, +::after { + box-sizing: border-box; + /* 1 */ + border-width: 0; + /* 2 */ + border-style: solid; + /* 2 */ + border-color: #e5e7eb; + /* 2 */ +} + +::before, +::after { + --tw-content: ''; +} + +/* +1. Use a consistent sensible line-height in all browsers. +2. Prevent adjustments of font size after orientation changes in iOS. +3. Use a more readable tab size. +4. Use the user's configured `sans` font-family by default. +5. Use the user's configured `sans` font-feature-settings by default. +6. Use the user's configured `sans` font-variation-settings by default. +*/ + +html { + line-height: 1.5; + /* 1 */ + -webkit-text-size-adjust: 100%; + /* 2 */ + -moz-tab-size: 4; + /* 3 */ + -o-tab-size: 4; + tab-size: 4; + /* 3 */ + font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; + /* 4 */ + font-feature-settings: normal; + /* 5 */ + font-variation-settings: normal; + /* 6 */ +} + +/* +1. Remove the margin in all browsers. +2. Inherit line-height from `html` so users can set them as a class directly on the `html` element. +*/ + +body { + margin: 0; + /* 1 */ + line-height: inherit; + /* 2 */ +} + +/* +1. Add the correct height in Firefox. +2. Correct the inheritance of border color in Firefox. (https://bugzilla.mozilla.org/show_bug.cgi?id=190655) +3. Ensure horizontal rules are visible by default. +*/ + +hr { + height: 0; + /* 1 */ + color: inherit; + /* 2 */ + border-top-width: 1px; + /* 3 */ +} + +/* +Add the correct text decoration in Chrome, Edge, and Safari. +*/ + +abbr:where([title]) { + -webkit-text-decoration: underline dotted; + text-decoration: underline dotted; +} + +/* +Remove the default font size and weight for headings. +*/ + +h1, +h2, +h3, +h4, +h5, +h6 { + font-size: inherit; + font-weight: inherit; +} + +/* +Reset links to optimize for opt-in styling instead of opt-out. +*/ + +a { + color: inherit; + text-decoration: inherit; +} + +/* +Add the correct font weight in Edge and Safari. +*/ + +b, +strong { + font-weight: bolder; +} + +/* +1. Use the user's configured `mono` font-family by default. +2. Use the user's configured `mono` font-feature-settings by default. +3. Use the user's configured `mono` font-variation-settings by default. +4. Correct the odd `em` font sizing in all browsers. +*/ + +code, +kbd, +samp, +pre { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + /* 1 */ + font-feature-settings: normal; + /* 2 */ + font-variation-settings: normal; + /* 3 */ + font-size: 1em; + /* 4 */ +} + +/* +Add the correct font size in all browsers. +*/ + +small { + font-size: 80%; +} + +/* +Prevent `sub` and `sup` elements from affecting the line height in all browsers. +*/ + +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} + +sub { + bottom: -0.25em; +} + +sup { + top: -0.5em; +} + +/* +1. Remove text indentation from table contents in Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=999088, https://bugs.webkit.org/show_bug.cgi?id=201297) +2. Correct table border color inheritance in all Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=935729, https://bugs.webkit.org/show_bug.cgi?id=195016) +3. Remove gaps between table borders by default. +*/ + +table { + text-indent: 0; + /* 1 */ + border-color: inherit; + /* 2 */ + border-collapse: collapse; + /* 3 */ +} + +/* +1. Change the font styles in all browsers. +2. Remove the margin in Firefox and Safari. +3. Remove default padding in all browsers. +*/ + +button, +input, +optgroup, +select, +textarea { + font-family: inherit; + /* 1 */ + font-feature-settings: inherit; + /* 1 */ + font-variation-settings: inherit; + /* 1 */ + font-size: 100%; + /* 1 */ + font-weight: inherit; + /* 1 */ + line-height: inherit; + /* 1 */ + color: inherit; + /* 1 */ + margin: 0; + /* 2 */ + padding: 0; + /* 3 */ +} + +/* +Remove the inheritance of text transform in Edge and Firefox. +*/ + +button, +select { + text-transform: none; +} + +/* +1. Correct the inability to style clickable types in iOS and Safari. +2. Remove default button styles. +*/ + +button, +[type='button'], +[type='reset'], +[type='submit'] { + -webkit-appearance: button; + /* 1 */ + background-color: transparent; + /* 2 */ + background-image: none; + /* 2 */ +} + +/* +Use the modern Firefox focus style for all focusable elements. +*/ + +:-moz-focusring { + outline: auto; +} + +/* +Remove the additional `:invalid` styles in Firefox. (https://github.com/mozilla/gecko-dev/blob/2f9eacd9d3d995c937b4251a5557d95d494c9be1/layout/style/res/forms.css#L728-L737) +*/ + +:-moz-ui-invalid { + box-shadow: none; +} + +/* +Add the correct vertical alignment in Chrome and Firefox. +*/ + +progress { + vertical-align: baseline; +} + +/* +Correct the cursor style of increment and decrement buttons in Safari. +*/ + +::-webkit-inner-spin-button, +::-webkit-outer-spin-button { + height: auto; +} + +/* +1. Correct the odd appearance in Chrome and Safari. +2. Correct the outline style in Safari. +*/ + +[type='search'] { + -webkit-appearance: textfield; + /* 1 */ + outline-offset: -2px; + /* 2 */ +} + +/* +Remove the inner padding in Chrome and Safari on macOS. +*/ + +::-webkit-search-decoration { + -webkit-appearance: none; +} + +/* +1. Correct the inability to style clickable types in iOS and Safari. +2. Change font properties to `inherit` in Safari. +*/ + +::-webkit-file-upload-button { + -webkit-appearance: button; + /* 1 */ + font: inherit; + /* 2 */ +} + +/* +Add the correct display in Chrome and Safari. +*/ + +summary { + display: list-item; +} + +/* +Removes the default spacing and border for appropriate elements. +*/ + +blockquote, +dl, +dd, +h1, +h2, +h3, +h4, +h5, +h6, +hr, +figure, +p, +pre { + margin: 0; +} + +fieldset { + margin: 0; + padding: 0; +} + +legend { + padding: 0; +} + +ol, +ul, +menu { + list-style: none; + margin: 0; + padding: 0; +} + +/* +Reset default styling for dialogs. +*/ + +dialog { + padding: 0; +} + +/* +Prevent resizing textareas horizontally by default. +*/ + +textarea { + resize: vertical; +} + +/* +1. Reset the default placeholder opacity in Firefox. (https://github.com/tailwindlabs/tailwindcss/issues/3300) +2. Set the default placeholder color to the user's configured gray 400 color. +*/ + +input::-moz-placeholder, textarea::-moz-placeholder { + opacity: 1; + /* 1 */ + color: #9ca3af; + /* 2 */ +} + +input::placeholder, +textarea::placeholder { + opacity: 1; + /* 1 */ + color: #9ca3af; + /* 2 */ +} + +/* +Set the default cursor for buttons. +*/ + +button, +[role="button"] { + cursor: pointer; +} + +/* +Make sure disabled buttons don't get the pointer cursor. +*/ + +:disabled { + cursor: default; +} + +/* +1. Make replaced elements `display: block` by default. (https://github.com/mozdevs/cssremedy/issues/14) +2. Add `vertical-align: middle` to align replaced elements more sensibly by default. (https://github.com/jensimmons/cssremedy/issues/14#issuecomment-634934210) + This can trigger a poorly considered lint error in some tools but is included by design. +*/ + +img, +svg, +video, +canvas, +audio, +iframe, +embed, +object { + display: block; + /* 1 */ + vertical-align: middle; + /* 2 */ +} + +/* +Constrain images and videos to the parent width and preserve their intrinsic aspect ratio. (https://github.com/mozdevs/cssremedy/issues/14) +*/ + +img, +video { + max-width: 100%; + height: auto; +} + +/* Make elements with the HTML hidden attribute stay hidden by default */ + +[hidden] { + display: none; +} + +a { + --tw-text-opacity: 1; + color: rgb(59 130 246 / var(--tw-text-opacity)); + text-decoration-line: underline; +} + +a:hover { + --tw-text-opacity: 1; + color: rgb(29 78 216 / var(--tw-text-opacity)); +} + +*, ::before, ::after { + --tw-border-spacing-x: 0; + --tw-border-spacing-y: 0; + --tw-translate-x: 0; + --tw-translate-y: 0; + --tw-rotate: 0; + --tw-skew-x: 0; + --tw-skew-y: 0; + --tw-scale-x: 1; + --tw-scale-y: 1; + --tw-pan-x: ; + --tw-pan-y: ; + --tw-pinch-zoom: ; + --tw-scroll-snap-strictness: proximity; + --tw-gradient-from-position: ; + --tw-gradient-via-position: ; + --tw-gradient-to-position: ; + --tw-ordinal: ; + --tw-slashed-zero: ; + --tw-numeric-figure: ; + --tw-numeric-spacing: ; + --tw-numeric-fraction: ; + --tw-ring-inset: ; + --tw-ring-offset-width: 0px; + --tw-ring-offset-color: #fff; + --tw-ring-color: rgb(59 130 246 / 0.5); + --tw-ring-offset-shadow: 0 0 #0000; + --tw-ring-shadow: 0 0 #0000; + --tw-shadow: 0 0 #0000; + --tw-shadow-colored: 0 0 #0000; + --tw-blur: ; + --tw-brightness: ; + --tw-contrast: ; + --tw-grayscale: ; + --tw-hue-rotate: ; + --tw-invert: ; + --tw-saturate: ; + --tw-sepia: ; + --tw-drop-shadow: ; + --tw-backdrop-blur: ; + --tw-backdrop-brightness: ; + --tw-backdrop-contrast: ; + --tw-backdrop-grayscale: ; + --tw-backdrop-hue-rotate: ; + --tw-backdrop-invert: ; + --tw-backdrop-opacity: ; + --tw-backdrop-saturate: ; + --tw-backdrop-sepia: ; +} + +::backdrop { + --tw-border-spacing-x: 0; + --tw-border-spacing-y: 0; + --tw-translate-x: 0; + --tw-translate-y: 0; + --tw-rotate: 0; + --tw-skew-x: 0; + --tw-skew-y: 0; + --tw-scale-x: 1; + --tw-scale-y: 1; + --tw-pan-x: ; + --tw-pan-y: ; + --tw-pinch-zoom: ; + --tw-scroll-snap-strictness: proximity; + --tw-gradient-from-position: ; + --tw-gradient-via-position: ; + --tw-gradient-to-position: ; + --tw-ordinal: ; + --tw-slashed-zero: ; + --tw-numeric-figure: ; + --tw-numeric-spacing: ; + --tw-numeric-fraction: ; + --tw-ring-inset: ; + --tw-ring-offset-width: 0px; + --tw-ring-offset-color: #fff; + --tw-ring-color: rgb(59 130 246 / 0.5); + --tw-ring-offset-shadow: 0 0 #0000; + --tw-ring-shadow: 0 0 #0000; + --tw-shadow: 0 0 #0000; + --tw-shadow-colored: 0 0 #0000; + --tw-blur: ; + --tw-brightness: ; + --tw-contrast: ; + --tw-grayscale: ; + --tw-hue-rotate: ; + --tw-invert: ; + --tw-saturate: ; + --tw-sepia: ; + --tw-drop-shadow: ; + --tw-backdrop-blur: ; + --tw-backdrop-brightness: ; + --tw-backdrop-contrast: ; + --tw-backdrop-grayscale: ; + --tw-backdrop-hue-rotate: ; + --tw-backdrop-invert: ; + --tw-backdrop-opacity: ; + --tw-backdrop-saturate: ; + --tw-backdrop-sepia: ; +} + +.container { + width: 100%; +} + +@media (min-width: 640px) { + .container { + max-width: 640px; + } +} + +@media (min-width: 768px) { + .container { + max-width: 768px; + } +} + +@media (min-width: 1024px) { + .container { + max-width: 1024px; + } +} + +@media (min-width: 1280px) { + .container { + max-width: 1280px; + } +} + +@media (min-width: 1536px) { + .container { + max-width: 1536px; + } +} + +.visible { + visibility: visible; +} + +.fixed { + position: fixed; +} + +.absolute { + position: absolute; +} + +.table { + display: table; +} + +.grid { + display: grid; +} + +.hidden { + display: none; +} + +.transform { + transform: translate(var(--tw-translate-x), var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y)); +} + +.invert { + --tw-invert: invert(100%); + filter: var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow); +} + +.filter { + filter: var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow); +} + +.transition { + transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, -webkit-backdrop-filter; + transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, backdrop-filter; + transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, backdrop-filter, -webkit-backdrop-filter; + transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1); + transition-duration: 150ms; +} diff --git a/workflow/templates/assets/css/tailwind.css b/workflow/templates/assets/css/tailwind.css new file mode 100644 index 0000000..b3a8920 --- /dev/null +++ b/workflow/templates/assets/css/tailwind.css @@ -0,0 +1,9 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +@layer base { + a { + @apply text-blue-500 hover:text-blue-700 underline; + } +} From b1dd4f962331a638828bc060e270820d26266f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20M=C3=A4hler?= <2573608+maehler@users.noreply.github.com> Date: Mon, 11 Dec 2023 17:18:44 +0100 Subject: [PATCH 11/12] Add a *very* rudimentary MultiQC table --- .../config/NA12878_N.general_report.json | 7 ++++ .../general_html_report_json.schema.yaml | 28 +++++++++++++ workflow/scripts/general_html_report.py | 39 ++++++++++++++++++- .../templates/general_html_report/index.html | 23 ++++++++++- 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/.tests/integration/config/NA12878_N.general_report.json b/.tests/integration/config/NA12878_N.general_report.json index b202764..9aa827e 100644 --- a/.tests/integration/config/NA12878_N.general_report.json +++ b/.tests/integration/config/NA12878_N.general_report.json @@ -50,6 +50,13 @@ "type": "image", "description": "A local image from a path relative to the analysis directory", "value": "test_data/NA12878_N.png" + }, + { + "name": "QC", + "type": "multiqc", + "description": "QC data from MultiQC with the other samples in the sampe run included for context", + "sections": ["table"], + "value": "qc/multiqc/multiqc_DNA_data/multiqc_data.json" } ] } diff --git a/workflow/schemas/general_html_report_json.schema.yaml b/workflow/schemas/general_html_report_json.schema.yaml index fa32d37..cba4b80 100644 --- a/workflow/schemas/general_html_report_json.schema.yaml +++ b/workflow/schemas/general_html_report_json.schema.yaml @@ -86,6 +86,34 @@ properties: - description - type - value + - type: object + properties: + name: + type: string + description: A descriptive name of the result + description: + type: string + description: A more detailed description of the result + type: + type: string + description: The type of result + const: multiqc + sections: + type: array + description: List of sections to include. + items: + type: string + description: Name of the section + enum: + - table + value: + description: Path to a MultiQC JSON file where this sample is included. + required: + - name + - description + - type + - sections + - value required: - sample diff --git a/workflow/scripts/general_html_report.py b/workflow/scripts/general_html_report.py index ad6a230..2ea0147 100644 --- a/workflow/scripts/general_html_report.py +++ b/workflow/scripts/general_html_report.py @@ -1,14 +1,21 @@ from jinja2 import Template import json from jsonschema import validate +from jsonschema.exceptions import ValidationError import re +import sys import time import yaml def validate_dict(d: dict, schema_path: str): with open(schema_path) as f: - validate(instance=d, schema=yaml.safe_load(f)) + try: + validate(instance=d, schema=yaml.safe_load(f)) + except ValidationError as ve: + print(f"error: failed to validate general report config:", file=sys.stderr) + print(ve, file=sys.stderr) + sys.exit(1) def validate_table_data(table: list) -> bool: @@ -37,6 +44,34 @@ def fix_relative_uri(uri: str, depth: int) -> str: return depth * "../" + uri +def parse_multiqc(d: dict): + with open(d["value"]) as f: + multiqc_dict = json.loads(f.read()) + + multiqc_res = {} + + for s in d["sections"]: + if s == "table": + multiqc_res[s] = {} + multiqc_res[s]["data"] = {} + multiqc_res[s]["header"] = {} + + for h_section, d_section in zip( + multiqc_dict["report_general_stats_headers"], multiqc_dict["report_general_stats_data"] + ): + for k, v in h_section.items(): + multiqc_res[s]["header"][k] = v + for sample, cols in d_section.items(): + if sample not in multiqc_res[s]["data"]: + multiqc_res[s]["data"][sample] = {} + for k, v in h_section.items(): + if k not in cols: + continue + multiqc_res[s]["data"][sample][k] = cols[k] + + return multiqc_res + + def generate_report(template_filename: str, config: dict, final_directory_depth: int) -> str: with open(template_filename) as f: template = Template(source=f.read()) @@ -49,6 +84,8 @@ def generate_report(template_filename: str, config: dict, final_directory_depth: validate_table_data(d["value"]) if d["type"] == "image": d["value"] = fix_relative_uri(d["value"], final_directory_depth) + if d["type"] == "multiqc": + d["data"] = parse_multiqc(d) return template.render( dict( diff --git a/workflow/templates/general_html_report/index.html b/workflow/templates/general_html_report/index.html index 3ebf70f..17ec058 100644 --- a/workflow/templates/general_html_report/index.html +++ b/workflow/templates/general_html_report/index.html @@ -76,7 +76,28 @@

    {{ r.name }}

    {{ r.value }}

    {% elif r.type == "image" %} - {% endif %} + {% elif r.type == "multiqc" %} {% if "table" in r.data %} + + + + + {% for k, v in r.data["table"]["header"].items() %} + + {% endfor %} + + + + {% for k, v in r.data["table"]["data"].items() %} + + + {% for hk in r.data["table"]["header"].keys() %} + + {% endfor %} + + {% endfor %} + +
    Sample{{ v.title }}
    {{ k }}{{ v[hk] }}
    + {% endif %} {% endif %} {% endfor %} From 72c4e25ade7dd73379b604a5f85fdb31cc8114c9 Mon Sep 17 00:00:00 2001 From: chels0 Date: Thu, 20 Jun 2024 15:33:14 +0200 Subject: [PATCH 12/12] feat: generate json for html report, report styling --- .tests/integration/config.yaml | 3 +- config/config.yaml | 3 +- config/general_report.yaml | 6 + config/output_files.yaml | 8 +- workflow/rules/common.smk | 9 +- workflow/rules/general_html_report.smk | 41 +++- workflow/schemas/config.schema.yaml | 10 - .../general_html_report_json.schema.yaml | 28 +-- workflow/schemas/general_report.schema.yaml | 35 +++ workflow/schemas/rules.schema.yaml | 4 + workflow/scripts/cnv_html_report.py | 6 +- workflow/scripts/cnv_json.py | 21 +- workflow/scripts/general_html_report.py | 99 +++++--- workflow/scripts/general_json_report.py | 119 +++++++++ workflow/scripts/merge_cnv_json.py | 89 ++++--- workflow/scripts/test_cnv_json.py | 3 +- .../templates/general_html_report/index.html | 227 +++++++++++------- .../templates/general_html_report/style.css | 181 ++++++++++++++ 18 files changed, 678 insertions(+), 214 deletions(-) create mode 100644 config/general_report.yaml create mode 100644 workflow/schemas/general_report.schema.yaml create mode 100644 workflow/scripts/general_json_report.py create mode 100644 workflow/templates/general_html_report/style.css diff --git a/.tests/integration/config.yaml b/.tests/integration/config.yaml index b138112..e3b4c99 100644 --- a/.tests/integration/config.yaml +++ b/.tests/integration/config.yaml @@ -1,4 +1,5 @@ output: "../../config/output_files.yaml" +general_report: "../../config/general_report.yaml" resources: "resources.yaml" samples: "samples.tsv" units: "units.tsv" @@ -10,9 +11,9 @@ cnv_html_report: cytobands: true general_html_report: - json: "config/{sample}_{type}.general_report.json" final_directory_depth: 2 + merge_cnv_json: annotations: - config/amp_genes.bed diff --git a/config/config.yaml b/config/config.yaml index da93564..100e967 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,5 +1,6 @@ --- output: "config/output_files.yaml" +general_report: "config/general_report.yaml" resources: "config/resources.yaml" samples: "samples.tsv" units: "units.tsv" @@ -15,7 +16,7 @@ cnv_html_report: show_table: true general_html_report: - json: "reports/general_html_report/{sample}_{type}.general_report.json" + final_directory_depth: 2 merge_cnv_json: annotations: diff --git a/config/general_report.yaml b/config/general_report.yaml new file mode 100644 index 0000000..88df0b4 --- /dev/null +++ b/config/general_report.yaml @@ -0,0 +1,6 @@ +files: + - name: CNV report + type: file_link + description: Link to CNV HTML report + input: results/dna/cnv/{sample}_{type}/{sample}_{type}.pathology_purecn.cnv.html + nav_header: CNV \ No newline at end of file diff --git a/config/output_files.yaml b/config/output_files.yaml index 8a836e9..16442c2 100644 --- a/config/output_files.yaml +++ b/config/output_files.yaml @@ -2,6 +2,10 @@ files: - name: "CNV HTML report" input: "reports/cnv_html_report/{sample}_{type}.pathology.cnv_report.html" output: "results/cnv/{sample}_{type}.pathology.cnv_report.html" - - name: "General HTML report" + types: + - T + - name: "General Report" input: "reports/general_html_report/{sample}_{type}.general_report.html" - output: "results/reports/{sample}_{type}.general_report.html" + output: "results/dna/reports/{sample}_{type}.general_report.html" + types: + - T diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 67d8610..da9acc1 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -13,7 +13,7 @@ import yaml from snakemake.io import Wildcards from snakemake.utils import validate from snakemake.utils import min_version - +from datetime import datetime from hydra_genetics.utils.resources import load_resources from hydra_genetics.utils.samples import * from hydra_genetics.utils.units import * @@ -45,6 +45,7 @@ units = ( validate(units, schema="../schemas/units.schema.yaml") + with open(config["output"]) as output: if config["output"].endswith("json"): output_spec = json.load(output) @@ -53,6 +54,12 @@ with open(config["output"]) as output: validate(output_spec, schema="../schemas/output_files.schema.yaml") +with open(config["general_report"]) as f: + if f.name.endswith(".yaml"): + general_report = yaml.safe_load(f) + +validate(general_report, "../schemas/general_report.schema.yaml") + ### Set wildcard constraints wildcard_constraints: diff --git a/workflow/rules/general_html_report.smk b/workflow/rules/general_html_report.smk index 135ab1a..547b511 100644 --- a/workflow/rules/general_html_report.smk +++ b/workflow/rules/general_html_report.smk @@ -1,14 +1,49 @@ -__author__ = "Niklas Mähler" +__author__ = "Niklas Mähler & Chelsea Ramsin" __copyright__ = "Copyright 2023, Niklas Mähler" -__email__ = "niklas.mahler@regionvasterbotten.se" +__email__ = "niklas.mahler@regionvasterbotten.se & chelsea.ramsin@regionostergotland.se" __license__ = "GPL-3" +rule general_json_report: + input: + files=[f'{filedef["input"]}' for filedef in general_report["files"]], + output_files=config.get("general_report", {}), + additional_metadata=[], + output: + out="reports/general_report/{sample}_{type}.general.json", + params: + sample="{sample}_{type}", + log: + "reports/general_json_report/{sample}_{type}.general_report.log", + benchmark: + repeat( + "reports/general_json_report/{sample}_{type}.output.benchmark.tsv", + config.get("general_json_report", {}).get("benchmark_repeats", 1), + ) + threads: config.get("general_json_report", {}).get("threads", config["default_resources"]["threads"]) + resources: + mem_mb=config.get("general_json_report", {}).get("mem_mb", config["default_resources"]["mem_mb"]), + mem_per_cpu=config.get("general_json_report", {}).get("mem_per_cpu", config["default_resources"]["mem_per_cpu"]), + partition=config.get("general_json_report", {}).get("partition", config["default_resources"]["partition"]), + threads=config.get("general_json_report", {}).get("threads", config["default_resources"]["threads"]), + time=config.get("general_json_report", {}).get("time", config["default_resources"]["time"]), + container: + config.get("general_json_report", {}).get("container", config["default_container"]) + message: + "{rule}: generate general html report from json config" + script: + "../scripts/general_json_report.py" + + rule general_html_report: input: config_schema=workflow.source_path("../schemas/general_html_report_json.schema.yaml"), html_template=workflow.source_path("../templates/general_html_report/index.html"), - json=config.get("general_html_report", {}).get("json"), + json="reports/general_report/{sample}_{type}.general.json", + css_files=[ + workflow.source_path("../templates/general_html_report/style.css"), + ], + additional_json={}, output: html="reports/general_html_report/{sample}_{type}.general_report.html", params: diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml index 2426efc..5e36abd 100644 --- a/workflow/schemas/config.schema.yaml +++ b/workflow/schemas/config.schema.yaml @@ -66,15 +66,6 @@ properties: type: object description: parameters for general_html_report properties: - json: - type: string - format: uri-reference - description: | - Path to the sample-specific configuration of the report. The wildcards - `sample` and `type` are supported. - examples: - - "report_configs/{sample}_{type}/general_report.json" - - "report_configs/general_html_report/{sample}_{type}.general_report.json" final_directory_depth: type: integer description: | @@ -92,7 +83,6 @@ properties: type: string description: parameters that should be forwarded required: - - json - final_directory_depth merge_cnv_json: diff --git a/workflow/schemas/general_html_report_json.schema.yaml b/workflow/schemas/general_html_report_json.schema.yaml index cba4b80..0c2a018 100644 --- a/workflow/schemas/general_html_report_json.schema.yaml +++ b/workflow/schemas/general_html_report_json.schema.yaml @@ -26,28 +26,6 @@ properties: format: uri-reference description: URI of the pipeline - file_links: - type: array - description: List of file links - default: [] - items: - type: object - properties: - name: - type: string - description: Descriptive name of the file - description: - type: string - description: Description of the file - uri: - type: string - format: uri-reference - description: URI of the file - required: - - name - - description - - uri - results: type: array description: List of results to present @@ -72,15 +50,18 @@ properties: - plain_text - single_value - table + - file_link value: description: | The value of the result. Exactly what this is depends on the type. - - file_table: a delimited text file (csv or tsv) with a header. - image: the path to a png image. - plain_text: a string containing the text to be presented. - single_value: a single value, e.g. a number. - table: a json representation of a table. + - file_link: path to file + nav_header: + description: If report is to be navigated through a tab menu, write under which header this result should fall under. For example Biomarker or Fusions. required: - name - description @@ -119,5 +100,4 @@ required: - sample - analysis_date - pipeline - - file_links - results diff --git a/workflow/schemas/general_report.schema.yaml b/workflow/schemas/general_report.schema.yaml new file mode 100644 index 0000000..5492f1b --- /dev/null +++ b/workflow/schemas/general_report.schema.yaml @@ -0,0 +1,35 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: General Report yaml Config +description: Configuration for a general HTML report for a Hydra Genetics pipeline +type: object +properties: + files: + type: array + description: Defines results for general report + items: + type: object + properties: + name: + type: string + description: A descriptive name of the result + description: + type: string + description: A more detailed description of the result + type: + type: string + description: The type of result + input: + type: string + description: Relative path to results file + nav_header: + type: string + description: Header for navigation bar in HTML report. For example Biomarker + required: + - name + - description + - type + - input + - nav_header + +required: + - files diff --git a/workflow/schemas/rules.schema.yaml b/workflow/schemas/rules.schema.yaml index b816a02..82c5dad 100644 --- a/workflow/schemas/rules.schema.yaml +++ b/workflow/schemas/rules.schema.yaml @@ -97,6 +97,10 @@ properties: html_template: type: string description: path to the html template to use for the report + internal_json: + type: [string, object] + description: pipeline specific json file + output: type: object diff --git a/workflow/scripts/cnv_html_report.py b/workflow/scripts/cnv_html_report.py index 86acf98..b34fca4 100644 --- a/workflow/scripts/cnv_html_report.py +++ b/workflow/scripts/cnv_html_report.py @@ -8,7 +8,8 @@ def get_sample_name(filename): return Path(filename).name.split(".")[0] -def create_report(template_filename, json_filename, css_files, js_files, show_table, tc, tc_method): +def create_report(template_filename, json_filename, css_files, js_files, + show_table, tc, tc_method): with open(template_filename) as f: template = Template(source=f.read()) @@ -37,8 +38,7 @@ def create_report(template_filename, json_filename, css_files, js_files, show_ta tc=tc, tc_method=tc_method, ), - ) - ) + )) def main(): diff --git a/workflow/scripts/cnv_json.py b/workflow/scripts/cnv_json.py index 635ffd3..9f78ed5 100644 --- a/workflow/scripts/cnv_json.py +++ b/workflow/scripts/cnv_json.py @@ -5,7 +5,6 @@ from pathlib import Path import sys - # The functions `parse_*_ratios` functions take a filename of a file containing # copy number log2-ratios across the genome for a specific CNV caller. The # functions `parse_*_segments` takes a filename of a file containing log2-ratio @@ -21,7 +20,6 @@ # } # - PARSERS = collections.defaultdict(dict) @@ -90,8 +88,7 @@ def parse_cnvkit_ratios(file): start=int(line[1]), end=int(line[2]), log2=float(line[5]), - ) - ) + )) return ratios @@ -105,8 +102,7 @@ def parse_cnvkit_segments(file): start=int(line[1]), end=int(line[2]), log2=float(line[4]), - ) - ) + )) return segments @@ -120,8 +116,7 @@ def parse_gatk_ratios(file): start=int(line[1]), end=int(line[2]), log2=float(line[3]), - ) - ) + )) return ratios @@ -135,8 +130,7 @@ def parse_gatk_segments(file): start=int(line[1]), end=int(line[2]), log2=float(line[4]), - ) - ) + )) return segments @@ -163,7 +157,8 @@ def main(): skip_chromosomes = snakemake.params["skip_chromosomes"] - csv.field_size_limit(snakemake.params.get('csv_field_size_limt', 100000000)) + csv.field_size_limit(snakemake.params.get('csv_field_size_limt', + 100000000)) if caller not in PARSERS: print(f"error: no parser for {caller} implemented", file=sys.stderr) @@ -174,7 +169,9 @@ def main(): if skip_chromosomes is not None: ratios = [r for r in ratios if r["chromosome"] not in skip_chromosomes] - segments = [s for s in segments if s["chromosome"] not in skip_chromosomes] + segments = [ + s for s in segments if s["chromosome"] not in skip_chromosomes + ] with open(output_filename, "w") as f: print(to_json(caller, ratios, segments), file=f) diff --git a/workflow/scripts/general_html_report.py b/workflow/scripts/general_html_report.py index 2ea0147..c65dd6b 100644 --- a/workflow/scripts/general_html_report.py +++ b/workflow/scripts/general_html_report.py @@ -6,14 +6,16 @@ import sys import time import yaml +import pandas as pd def validate_dict(d: dict, schema_path: str): - with open(schema_path) as f: + with open(schema_path, 'r') as f: try: validate(instance=d, schema=yaml.safe_load(f)) except ValidationError as ve: - print(f"error: failed to validate general report config:", file=sys.stderr) + print(f"error: failed to validate general report config:", + file=sys.stderr) print(ve, file=sys.stderr) sys.exit(1) @@ -27,20 +29,18 @@ def validate_table_data(table: list) -> bool: for i, row in enumerate(table, start=1): if len(row) != n_cols: - raise ValueError(f"expected {n_cols} columns in row {i}, found {len(row)}") + raise ValueError( + f"expected {n_cols} columns in row {i}, found {len(row)}") if set(row.keys()) != set(cols): raise ValueError( f"expected columns {', '.join(repr(x) for x in cols)} " - f"in row {i}, found {', '.join(repr(x) for x in row.keys())}" - ) - + f"in row {i}, found {', '.join(repr(x) for x in row.keys())}") return True def fix_relative_uri(uri: str, depth: int) -> str: if re.match("^([^/]+:|/)", uri): return uri - return depth * "../" + uri @@ -57,8 +57,8 @@ def parse_multiqc(d: dict): multiqc_res[s]["header"] = {} for h_section, d_section in zip( - multiqc_dict["report_general_stats_headers"], multiqc_dict["report_general_stats_data"] - ): + multiqc_dict["report_general_stats_headers"], + multiqc_dict["report_general_stats_data"]): for k, v in h_section.items(): multiqc_res[s]["header"][k] = v for sample, cols in d_section.items(): @@ -68,50 +68,93 @@ def parse_multiqc(d: dict): if k not in cols: continue multiqc_res[s]["data"][sample][k] = cols[k] - return multiqc_res -def generate_report(template_filename: str, config: dict, final_directory_depth: int) -> str: +def navigation_bar(config: dict): + headers = [] + for d in config["results"]: + headers.append(d["nav_header"]) + headers_no_dup = list(set(headers)) + return headers_no_dup + + +def merge_json(config: dict, extra_config: dict): + for d in extra_config['results']: + config['results'].append(d) + return config + + +def generate_report(template_filename: str, config: dict, + final_directory_depth: int, css_files: list, + navigation_bar: list) -> str: with open(template_filename) as f: template = Template(source=f.read()) - for d in config["file_links"]: - d["uri"] = fix_relative_uri(d["uri"], final_directory_depth) - for d in config["results"]: + if d["type"] == "file_link": + d["value"] = fix_relative_uri(d["value"], final_directory_depth) if d["type"] == "table": validate_table_data(d["value"]) if d["type"] == "image": d["value"] = fix_relative_uri(d["value"], final_directory_depth) + if d["type"] == "multiqc": d["data"] = parse_multiqc(d) + if d["type"] == "file_table": + data = pd.read_csv(d['value'], sep='\t') + if len(data.columns) == 1: + data = pd.read_csv(d['value'], sep=',') + d["data"] = data.to_html(index=False).replace('border="1"', '') + + css_string = "" + for css_filename in css_files: + with open(css_filename) as f: + css_string += f.read() + + navigation_bar.sort() + nav_bar_html = '' + + # Rewrite in javascript + for header in navigation_bar: + nav_bar_html += f'\t\t\t\t\n' + return template.render( - dict( - metadata=dict( - analysis_date=config["analysis_date"], - report_date=time.strftime("%Y-%m-%d %H:%M", time.localtime()), - sample=config["sample"], - ), - pipeline=config["pipeline"], - file_links=config["file_links"], - results=config["results"], - ) - ) + dict(metadata=dict( + analysis_date=config["analysis_date"], + report_date=time.strftime("%Y-%m-%d %H:%M", time.localtime()), + sample=config["sample"], + ), + pipeline=config["pipeline"], + results=config["results"], + css=css_string, + nav_bar=nav_bar_html, + nav_header=navigation_bar)) def main(): html_template = snakemake.input.html_template json_file = snakemake.input.json + additional_json_file = snakemake.input.additional_json + css = snakemake.input.css_files + config_schema = snakemake.input.config_schema final_directory_depth = snakemake.params.final_directory_depth with open(json_file) as f: config = json.load(f) - validate_dict(config, snakemake.input.config_schema) - - report_content = generate_report(html_template, config, final_directory_depth) + if len(additional_json_file) == 0: + config = config + else: + with open(additional_json_file) as f: + additional_config = json.load(f) + config = merge_json(config, additional_config) + + nav_bar = navigation_bar(config) + validate_dict(config, schema_path=config_schema) + report_content = generate_report(html_template, config, + final_directory_depth, css, nav_bar) with open(snakemake.output.html, "w") as f: f.write(report_content) diff --git a/workflow/scripts/general_json_report.py b/workflow/scripts/general_json_report.py new file mode 100644 index 0000000..9f1552f --- /dev/null +++ b/workflow/scripts/general_json_report.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import yaml +import json +import csv + + +def metadata(sample): + # Still in progress + analysis_date = 'No date' + pipeline = dict(name='', version='', uri="") + dict_ = dict(sample=sample, analysis_date=analysis_date, pipeline=pipeline) + return dict_ + + +def table(file, name, type, description, nav_header): + with open(file) as f: + list_ = [] + csv_table = csv.reader(f, delimiter='\t') + header = next(csv_table) + table = zip(*csv_table) + for tupl in table: + for val in tupl: + list_.append(val) + table_dict = dict(zip(header, list_)) + dict_ = dict(name=name, + type=type, + description=description, + value=[table_dict], + nav_header=nav_header) + return dict_ + + +def file_table(file, name, type, description, nav_header): + dict_ = dict(name=name, + type=type, + description=description, + value=file, + nav_header=nav_header) + return dict_ + + +def multiqc(file, name, type, description, sections, nav_header): + dict_ = dict(name=name, + type=type, + description=description, + value=file, + sections=sections, + nav_header=nav_header) + return dict_ + + +def check_nav_header(files): + if files['nav_header']: + nav_header = files['nav_header'] + else: + nav_header = '' + return nav_header + + +def tmb(file, name, type, description, nav_header): + with open(file) as f: + tmb_report = f.readlines() + tmb_value = tmb_report[0] + nr_of_variants = tmb_report[1] + tmb_results = tmb_value + '
    ' + nr_of_variants + dict_ = dict(name=name, + type=type, + description=description, + value=tmb_results, + nav_header=nav_header) + return dict_ + + +def generate_json(output_files, sample): + with open(output_files, 'r') as f: + output_files = yaml.safe_load(f) + results = [] + for d in output_files['files']: + path = d['input'] + sample_path = path.replace('{sample}_{type}', sample) + + nav_header = check_nav_header(d) + if 'tmb' in d['input']: + results1 = tmb(sample_path, d['name'], d['type'], d['description'], + nav_header) + else: + if d['type'] == 'table': + results1 = table(sample_path, d['name'], d['type'], + d['description'], nav_header) + if d['type'] == 'file_table': + results1 = file_table(sample_path, d['name'], d['type'], + d['description'], nav_header) + if d['type'] == 'file_link': + results1 = file_table(sample_path, d['name'], d['type'], + d['description'], nav_header) + if d['type'] == 'multiqc': + results1 = multiqc(sample_path, d['name'], d['type'], + d['description'], ["table"], nav_header) + if d["type"] == "image": + results1 = file_table(sample_path, d['name'], d['type'], + d['description'], nav_header) + results.append(results1) + + meta_data = metadata(sample) + json_file = dict(meta_data, results=results) + return json_file + + +if __name__ == "__main__": + json_filename = snakemake.input.files + output = snakemake.output.out + output_files = snakemake.input.output_files + additional_metadata = snakemake.input.additional_metadata + sample = snakemake.params.sample + json_file = generate_json(output_files, sample) + with open(output, 'w') as f: + print(json.dumps(json_file), file=f) diff --git a/workflow/scripts/merge_cnv_json.py b/workflow/scripts/merge_cnv_json.py index 206d726..1e59261 100644 --- a/workflow/scripts/merge_cnv_json.py +++ b/workflow/scripts/merge_cnv_json.py @@ -25,20 +25,21 @@ def end(self): def overlaps(self, other): return self.chromosome == other.chromosome and ( # overlaps in the beginning, or self contained in other - (self.start >= other.start and self.start <= other.end()) - or + (self.start >= other.start and self.start <= other.end()) or # overlaps at the end, or self contained in other - (self.end() >= other.start and self.end() <= other.end()) - or + (self.end() >= other.start and self.end() <= other.end()) or # other is contained in self - (other.start >= self.start and other.end() <= self.end()) - ) + (other.start >= self.start and other.end() <= self.end())) def __hash__(self): - return hash(f"{self.caller}_{self.chromosome}:{self.start}-{self.end()}_{self.copy_number}") + return hash( + f"{self.caller}_{self.chromosome}:{self.start}-{self.end()}_{self.copy_number}" + ) -cytoband_config = snakemake.config.get("merge_cnv_json", {}).get("cytoband_config", {}).get("colors", {}) +cytoband_config = snakemake.config.get("merge_cnv_json", + {}).get("cytoband_config", + {}).get("colors", {}) cytoband_centromere = "acen" cytoband_colors = { "gneg": cytoband_config.get("gneg", "#e3e3e3"), @@ -81,25 +82,27 @@ def parse_cytobands(filename, skip=None): chrom, start, end, name, giemsa = line.strip().split() if skip is not None and chrom in skip: continue - cytobands[chrom].append( - { - "name": name, - "start": int(start), - "end": int(end), - "direction": "none", - "giemsa": giemsa, - "color": cytoband_color(giemsa), - } - ) + cytobands[chrom].append({ + "name": name, + "start": int(start), + "end": int(end), + "direction": "none", + "giemsa": giemsa, + "color": cytoband_color(giemsa), + }) for k, v in cytobands.items(): cytobands[k] = sorted(v, key=lambda x: x["start"]) - centromere_index = [i for i, x in enumerate(cytobands[k]) if x["giemsa"] == cytoband_centromere] + centromere_index = [ + i for i, x in enumerate(cytobands[k]) + if x["giemsa"] == cytoband_centromere + ] if len(centromere_index) > 0 and len(centromere_index) != 2: print( - f"error: chromosome {k} does not have 0 or 2 centromere bands, " f"found {len(centromere_index)}", file=sys.stderr - ) + f"error: chromosome {k} does not have 0 or 2 centromere bands, " + f"found {len(centromere_index)}", + file=sys.stderr) sys.exit(1) elif len(centromere_index) == 0: continue @@ -110,7 +113,8 @@ def parse_cytobands(filename, skip=None): return cytobands -def get_vaf(vcf_filename: Union[str, bytes, Path], skip=None) -> Generator[tuple, None, None]: +def get_vaf(vcf_filename: Union[str, bytes, Path], + skip=None) -> Generator[tuple, None, None]: vcf = pysam.VariantFile(str(vcf_filename)) for variant in vcf.fetch(): if variant.chrom in skip: @@ -126,7 +130,9 @@ def get_cnvs(vcf_filename, skip=None): continue caller = variant.info.get("CALLER") if caller is None: - raise KeyError("could not find caller information for variant, has the vcf been annotated?") + raise KeyError( + "could not find caller information for variant, has the vcf been annotated?" + ) genes = variant.info.get("Genes") if genes is None: continue @@ -146,7 +152,8 @@ def get_cnvs(vcf_filename, skip=None): return cnvs -def merge_cnv_dicts(dicts, vaf, annotations, cytobands, chromosomes, filtered_cnvs, unfiltered_cnvs): +def merge_cnv_dicts(dicts, vaf, annotations, cytobands, chromosomes, + filtered_cnvs, unfiltered_cnvs): callers = list(map(lambda x: x["caller"], dicts)) caller_labels = dict( cnvkit="cnvkit", @@ -160,7 +167,14 @@ def merge_cnv_dicts(dicts, vaf, annotations, cytobands, chromosomes, filtered_cn length=chrom_length, vaf=[], annotations=[], - callers={c: dict(name=c, label=caller_labels.get(c, c), ratios=[], segments=[], cnvs=[]) for c in callers}, + callers={ + c: dict(name=c, + label=caller_labels.get(c, c), + ratios=[], + segments=[], + cnvs=[]) + for c in callers + }, ) for a in annotations: @@ -170,20 +184,17 @@ def merge_cnv_dicts(dicts, vaf, annotations, cytobands, chromosomes, filtered_cn start=item[1], end=item[2], name=item[3], - ) - ) + )) for c in cytobands: cnvs[c]["cytobands"] = cytobands[c] if vaf is not None: for v in vaf: - cnvs[v[0]]["vaf"].append( - dict( - pos=v[1], - vaf=v[2], - ) - ) + cnvs[v[0]]["vaf"].append(dict( + pos=v[1], + vaf=v[2], + )) # Iterate over the unfiltered CNVs and pair them according to overlap. for uf_cnvs, f_cnvs in zip(unfiltered_cnvs, filtered_cnvs): @@ -227,8 +238,7 @@ def merge_cnv_dicts(dicts, vaf, annotations, cytobands, chromosomes, filtered_cn cn=c.copy_number, baf=c.baf, passed_filter=pass_filter, - ) - ) + )) added_cnvs.add(c) for d in dicts: @@ -238,16 +248,14 @@ def merge_cnv_dicts(dicts, vaf, annotations, cytobands, chromosomes, filtered_cn start=r["start"], end=r["end"], log2=r["log2"], - ) - ) + )) for s in d["segments"]: cnvs[s["chromosome"]]["callers"][d["caller"]]["segments"].append( dict( start=s["start"], end=s["end"], log2=s["log2"], - ) - ) + )) for v in cnvs.values(): v["callers"] = list(v["callers"].values()) @@ -300,7 +308,8 @@ def main(): filtered_cnv_vcfs.append(get_cnvs(f_vcf, skip_chromosomes)) unfiltered_cnv_vcfs.append(get_cnvs(uf_vcf, skip_chromosomes)) - cnvs = merge_cnv_dicts(cnv_dicts, vaf, annotations, cytobands, fai, filtered_cnv_vcfs, unfiltered_cnv_vcfs) + cnvs = merge_cnv_dicts(cnv_dicts, vaf, annotations, cytobands, fai, + filtered_cnv_vcfs, unfiltered_cnv_vcfs) with open(output_file, "w") as f: print(json.dumps(cnvs), file=f) diff --git a/workflow/scripts/test_cnv_json.py b/workflow/scripts/test_cnv_json.py index d93c5f5..a826107 100644 --- a/workflow/scripts/test_cnv_json.py +++ b/workflow/scripts/test_cnv_json.py @@ -22,4 +22,5 @@ def test_existing_parsers(): def test_parse_cnvkit_segments(cnvkit_segment_file): segments = cnv_json.PARSERS["cnvkit"]["segments"](cnvkit_segment_file) assert len(segments) == 9 - assert all(x in segments[0] for x in ["chromosome", "start", "end", "log2"]) + assert all(x in segments[0] + for x in ["chromosome", "start", "end", "log2"]) diff --git a/workflow/templates/general_html_report/index.html b/workflow/templates/general_html_report/index.html index 17ec058..963b2cb 100644 --- a/workflow/templates/general_html_report/index.html +++ b/workflow/templates/general_html_report/index.html @@ -1,28 +1,39 @@ - - {{ metadata.sample }} – General Report - - - - - -
    -

    {{ metadata.sample }}

    -

    Report generated at {{ metadata.report_date }}

    -

    Sample analysed at {{ metadata.analysis_date }}

    - -

    - - hydra-genetics/reports v0.2.0 - -

    - -
    - -
    + + + + {{ metadata.sample }} – General Report + + + + + + +
    +

    Sample: {{ metadata.sample }}

    + + + {{ nav_bar }} +
    + +
    +
    +

    {{ metadata.sample }}

    + +

    Report generated at {{ metadata.report_date }}

    +

    Sample analysed at {{ metadata.analysis_date }}

    + +

    + + hydra-genetics/reports v0.2.0 + +

    +

    Pipeline

    • Name: {{ pipeline.name }}
      • @@ -34,72 +45,112 @@

      Pipeline

    -
    -

    File links

    + {% for k in nav_header %} +
    +

    {{ metadata.sample }}

    +

    {{ k }} results

    - {% for fl in file_links %} -
    -

    {{ fl.name }}

    -

    {{ fl.description }}

    -

    {{ fl.uri.lstrip("./") }}

    -
    - {% endfor %} -
    + {% for r in results %} + {% if r.nav_header == k %} +
    +
    +

    {{ r.name }}

    +

    {{ r.description }}

    + {% if r.type == "table" %} + + + + {% for c in r.value[0].keys() %} + + {% endfor %} + + + + {% for row in r.value %} + + {% for k in r.value[0].keys() %} + + {% endfor %} + + {% endfor %} + +
    {{ c }}
    {{ row[k] }}
    + + {% elif r.type == "file_link" %} + {{ r.value }} + + {% elif r.type == "single_value" %} +
    +

    {{ r.value }}

    +
    + {% elif r.type == "image" %} + + + {% elif r.type == "file_table" %} +
    + {{ r.data }} +
    + {% elif r.type == "multiqc" %} {% if "table" in r.data %} +
    + + + + + {% for k, v in r.data["table"]["header"].items() %} + + {% endfor %} + + + + {% for k, v in r.data["table"]["data"].items() %} + + + {% for hk in r.data["table"]["header"].keys() %} + + {% endfor %} + + {% endfor %} + +
    Sample{{ v.title }}
    {{ k }}{{ v[hk] }}
    +
    + + {% endif %} {% endif %} +
    + {% endif %} + {% endfor %} +
    + {% endfor %} -
    -

    Results

    - - {% for r in results %} -
    -

    {{ r.name }}

    -

    {{ r.description }}

    - {% if r.type == "table" %} - - - - {% for c in r.value[0].keys() %} - - {% endfor %} - - - - {% for row in r.value %} - - {% for k in r.value[0].keys() %} - - {% endfor %} - - {% endfor %} - -
    {{ c }}
    {{ row[k] }}
    - {% elif r.type == "single_value" %} -

    {{ r.value }}

    - {% elif r.type == "image" %} - - {% elif r.type == "multiqc" %} {% if "table" in r.data %} - - - - - {% for k, v in r.data["table"]["header"].items() %} - - {% endfor %} - - - - {% for k, v in r.data["table"]["data"].items() %} - - - {% for hk in r.data["table"]["header"].keys() %} - - {% endfor %} - - {% endfor %} - -
    Sample{{ v.title }}
    {{ k }}{{ v[hk] }}
    - {% endif %} {% endif %} + +
    - {% endfor %} + + + + + +
    - - + + \ No newline at end of file diff --git a/workflow/templates/general_html_report/style.css b/workflow/templates/general_html_report/style.css new file mode 100644 index 0000000..651893f --- /dev/null +++ b/workflow/templates/general_html_report/style.css @@ -0,0 +1,181 @@ +* {box-sizing: border-box} + +.center { + margin-left: 380px; /*this will center your wrapper*/ + background-color: rgb(255, 255, 255); + padding-left:70px; + padding-right:70px; + padding-top: 50px; +} + +html, body{ + background-color: rgba(86, 118, 124, 0.045); +} + +h1 { + font-family: Arial, Helvetica, sans-serif; + color: rgb(8, 184, 204); + font-size: 40px; +} + + +h2 { + font-family: Arial, Helvetica, sans-serif; + color: rgb(4, 129, 174); + font-size: 25px; +} + +h3 { + font-family: Arial, Helvetica, sans-serif; + color: rgb(4, 129, 174); + font-size: 21px; +} + +li { + font-family: Arial, Helvetica, sans-serif; +} + +p { + font-family: Arial, Helvetica, sans-serif; + color: rgb(0, 0, 0); +} + +th { + font-family: Arial, Helvetica, sans-serif; + color: rgb(0, 0, 0); + background-color: rgba(161, 220, 242, 0.5); + text-align: left; + +} + + + +table, th, td, tbody { + font-family: Arial, Helvetica, sans-serif; + overflow-x:auto; + width: 300px; + +} + +thead { + display: table-header-group; + vertical-align: middle; + border-color: inherit; + border-collapse: separate; + box-shadow: 0 0 3.5px rgba(0, 0, 0, 0.15); + +} + +tr { + box-shadow: 0 0 3.5px rgba(0, 0, 0, 0.15); + border-radius: 25px; + border-collapse: separate; + border-left: 0; + border-radius: 4px; + border-spacing: 0px; +} + + +table { + font-family: Arial, Helvetica, sans-serif; + + } + + +hr { + color:rgb(0, 0, 0) +} + + +th, td { + padding: 10px; + box-shadow: 0 0 3.5px rgba(0, 0, 0, 0.15); + +} + +.test { + padding-bottom: 40px; + +} + +/* Style the tab */ +/*.tab { + float: left; + background-color: #f1f1f1; + width: 23%; + height: 100%; +} +*/ + +.tab { + height: 100%; /* Full-height: remove this if you want "auto" height */ + width: 400px; /* Set the width of the sidebar */ + position: fixed; /* Fixed Sidebar (stay in place on scroll) */ + z-index: 1; /* Stay on top */ + top: 0; /* Stay at the top */ + left: 0; + background-color: #f1f1f1;; /* Black */ + overflow-x: hidden; /* Disable horizontal scroll */ + padding-top: 20px; + + } + +/* Style the buttons that are used to open the tab content */ +.tab button { + display: block; + background-color: inherit; + color: black; + padding: 22px 16px; + width: 400px; + border: none; + outline: none; + text-align: left; + cursor: pointer; + transition: 0.3s; +} + +/* Change background color of buttons on hover */ +.tab button:hover { + background-color: #ddd; +} + +/* Create an active/current "tab button" class */ +.tab button.active { + background-color: #ccc; +} + +/* Style the tab content */ +.tabcontent { + background-color: white; + padding-bottom: 1000px; +} + +.scroll { + overflow-x:auto; + overflow-y: auto; +} + +.single p { + background-color: rgba(161, 220, 242, 0.5); + font-weight: bold; + padding: 15px; + box-shadow: 0 0 3.5px rgba(0, 0, 0, 0.15); + border-radius: 25px; + border-collapse: separate; + border-left: 0; + border-radius: 4px; + border-spacing: 0px; + vertical-align: middle; + border-color: inherit; + border-collapse: separate; +} + +div { + margin: 0 auto; +} + +@media screen and (max-width:1500px) { + .tab {width: 150px;} + .tab button {width: 150px;} + .center {margin-left: 150px;} +} \ No newline at end of file