From 03c8628c9a74cac1d9775302b588422535e9e241 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 22 Feb 2019 13:08:01 +0100 Subject: [PATCH 01/83] Added custid, app_tag; attempt 1 --- microSALT/store/lims_fetcher.py | 15 +++++++++++++-- microSALT/store/orm_models.py | 2 ++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 4ecbbf9a..fa926e26 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -19,7 +19,14 @@ def __init__(self, config, log): def load_lims_project_info(self, cg_projid): project = Project(self.lims, id=cg_projid) + samplelist = self.samples_in_project(cg_projid) + custids = list() + for sample in samplelist: + self.load_lims_sample_info(sample) + custids.append(self.data['Customer_ID']) + if not custids[1:] == custids[:-1]: + raise Exception("Project {} contains multiple Customer IDs".format(cg_projid)) try: #Resolves old format if ' ' in project.name: @@ -31,7 +38,8 @@ def load_lims_project_info(self, cg_projid): newdate = datetime(int(tmp[0]), int(tmp[1]), int(tmp[2])) self.data.update({'date_received': newdate, 'CG_ID_project': cg_projid, - 'Customer_ID_project' : realname}) + 'Customer_ID_project' : realname, + 'Customer_ID': custids[0]}) except KeyError as e: self.logger.warn("Unable to fetch LIMS info for project {}\nSource: {}".format(cg_projid, str(e))) @@ -94,7 +102,10 @@ def load_lims_sample_info(self, cg_sampleid, external=False): 'CG_ID_sample': sample.id, 'Customer_ID_sample' : sample.name, 'organism' : organism, - 'priority' : sample.udf['priority']}) + 'priority' : sample.udf['priority'], + 'Customer_ID': sample.udf['customer'], + 'application_tag': sample.udf['Sequencing Analysis'] +}) except KeyError as e: self.logger.warn("Unable to fetch LIMS info for sample {}. Review LIMS data.\nSource: {}"\ .format(cg_sampleid, str(e))) diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index e158be87..064b3cfa 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -29,6 +29,7 @@ class Samples(db.Model): n50 = db.Column(db.Integer, default=-1) contigs = db.Column(db.Integer, default=-1) priority = db.Column(db.String(20)) + application_tag = db.Column(db.String(15)) class Seq_types(db.Model): __tablename__ = 'seq_types' @@ -77,6 +78,7 @@ class Projects(db.Model): CG_ID_project = db.Column(db.String(15), primary_key=True, nullable=False) Customer_ID_project = db.Column(db.String(15)) date_ordered = db.Column(db.DateTime) + Customer_ID = db.Column(db.String(15)) class Versions(db.Model): __tablename__ = 'versions' From 423cadb8f05e9f6a34ae0a4f9a6dfe8bcbe2033d Mon Sep 17 00:00:00 2001 From: Emma Sernstad Date: Fri, 22 Feb 2019 13:25:28 +0100 Subject: [PATCH 02/83] Added cards for contactinformation and table for samples --- microSALT/server/templates/bootstrap.html | 41 ++++ microSALT/server/templates/layout.html | 59 +++++- microSALT/server/templates/report_page.html | 197 ++++++++++++++++---- 3 files changed, 247 insertions(+), 50 deletions(-) create mode 100644 microSALT/server/templates/bootstrap.html diff --git a/microSALT/server/templates/bootstrap.html b/microSALT/server/templates/bootstrap.html new file mode 100644 index 00000000..b83a1830 --- /dev/null +++ b/microSALT/server/templates/bootstrap.html @@ -0,0 +1,41 @@ +{% macro css() %} + +{% endmacro %} + +{% macro custom_css() %} + +{% endmacro %} diff --git a/microSALT/server/templates/layout.html b/microSALT/server/templates/layout.html index 7a39f719..038cfaa7 100644 --- a/microSALT/server/templates/layout.html +++ b/microSALT/server/templates/layout.html @@ -1,4 +1,5 @@ + @@ -9,8 +10,49 @@ name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> - @@ -20,25 +62,26 @@ + + -
- + {% block body %}{% endblock %} - +
- - + + @@ -46,9 +89,7 @@ var d = new Date(); document.getElementById("date").innerHTML = d.toDateString(); - + - - diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index 45b3ce17..e24939a0 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -12,57 +12,141 @@



Clinical Microbiology

{{date}}

-

Project Summary

-
- - - - - - - -
Project ID (CG Project ID) - {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Analysis date{{samples[0].date_analysis.date()}}
+ + +
+
+
+
+

Kontakt Clinical genomics

+
+ Clinical Genomics
+ Science for Life Laboratory
+ Tomtebodavägen 23
+ 171 65 Solna
+ P: (08) 524 81 500 +
+
+
+
+
+
+
+

Kontakt Kund

+
+ Kund {{custxxx}}
+ statusDB
+ statusDB
+ statusDB
+ P: 08-xxxxxx +
+
+
+
+
-

Sample Summary

-
- +
+
+

Project Summary

+
+ + + + + + + + + + + + + + +
Project ID (CG Project ID) + {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Customer ID (CG Project ID) + *
Analysis date{{samples[0].date_analysis.date()}}
Pipeline version{{build}}
+
+
+ + + +
+

Sample Summary

+
+ - - - + + + + + + {% for sample in samples %} - {% if sample.organism is not none %} - - {% else %} - - {% endif %} - {% if sample.ST_status == 'None' or sample.ST == -1 %} - - {% elif sample.ST <= -4 %} - - {% else %} - - {% endif %} - {% if sample.threshold == 'Failed' %} - - {% else %} - - {% endif %} - - {% endfor %} -
Sample ID CG Sample IDOrganismSequence TypeThresholdsApplikations tagAnkomstdatumBiblioteksprep datumBiblioteksmetodSekvensering datumSekvenseringsmetod
{{sample.Customer_ID_sample}} {{sample.CG_ID_sample}}{{sample.organism.replace('_', ' ').capitalize()}}Control{{ sample.ST_status }}{{sample.ST}} (Novel){{sample.ST_status }}{{sample.threshold }}{{sample.threshold }}
-
+ * + {{samples[0].projects.date_ordered.date()}} + * + * + * + * + + {% endfor %} + +
- + + + + + +
+
+

Summary of Results

+
+ + + + + + + + + {% for sample in samples %} + + + + {% if sample.organism is not none %} + + {% else %} + + {% endif %} + {% if sample.ST_status == 'None' or sample.ST == -1 %} + + {% elif sample.ST <= -4 %} + + {% else %} + + {% endif %} + {% if sample.threshold == 'Failed' %} + + {% else %} + + {% endif %} + + {% endfor %} +
Sample IDCG Sample IDOrganismSequence TypeThresholds
{{sample.Customer_ID_sample}}{{sample.CG_ID_sample}}{{sample.organism.replace('_', ' ').capitalize()}}Control{{ sample.ST_status }}{{sample.ST}} (Novel){{sample.ST_status }}{{sample.threshold }}{{sample.threshold }}
+
+
+
+
{% for sample in samples %}
@@ -143,9 +227,10 @@

Assembly

{% if sample.genome_length > -1 %} +
-
+
{% if not sample.seq_types|length == 0 and 'Invalid' not in sample.ST_status%} @@ -193,7 +278,7 @@

Resistance

- {% for seq_type in sample.resistances if seq_type.threshold == 'Passed' %} + {% for seq_type in sample.resistances if seq_type.threshold == 'Passed' %} {{loop.index}} {{seq_type.gene}} @@ -226,6 +311,36 @@

Resistance

{% endif %} {% endfor %} + +
+
+ +

Teknisk beskrivninga av analysen : "{{applicationtag}}"

+

+
+ +

+ +

Begränsningar av analysen : "{{applicationtag}}"

+

+
+ +

+ +

Signatur för godkännande av rapport

+

+ Valtteri Wirta
+ Head of unit, Clinical Genomics +

+ +
Slut på rapport
+ + +
+
+
+ + {% endblock %} From 51e2a293cb0b27b07717d6ecacc701178e73682c Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 22 Feb 2019 13:42:44 +0100 Subject: [PATCH 03/83] Job creatore uploads new fields --- microSALT/utils/job_creator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 53dfb228..7d899b47 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -200,6 +200,7 @@ def create_project(self, name): proj_col['CG_ID_project'] = name proj_col['Customer_ID_project'] = self.lims_fetcher.data['Customer_ID_project'] proj_col['date_ordered'] = self.lims_fetcher.data['date_received'] + proj_col['Customer_ID'] = self.lims_fetcher.data['Customer_ID'] self.db_pusher.add_rec(proj_col, 'Projects') def create_sample(self, name): @@ -212,6 +213,7 @@ def create_sample(self, name): sample_col['Customer_ID_sample'] = self.lims_fetcher.data['Customer_ID_sample'] sample_col["date_analysis"] = self.dt sample_col['organism']=self.lims_fetcher.data['organism'] + sample_col["application_tag"] = self.lims_fetcher.data['application_tag'] #self.db_pusher.purge_rec(sample_col['CG_ID_sample'], 'sample') self.db_pusher.add_rec(sample_col, 'Samples') except Exception as e: From c4f6e5029f28ef88daca381e8223b3ef0f54c962 Mon Sep 17 00:00:00 2001 From: Emma Sernstad Date: Wed, 27 Feb 2019 14:14:50 +0100 Subject: [PATCH 04/83] Importing some LIMS data to tables --- microSALT/server/templates/report_page.html | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index e24939a0..2799857f 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -18,7 +18,7 @@

-

Kontakt Clinical genomics

+

Kontakt Clinical genomics:

Clinical Genomics
Science for Life Laboratory
@@ -32,13 +32,11 @@

Kontakt Clinical genomics

-

Kontakt Kund

+

Kund:

- Kund {{custxxx}}
- statusDB
- statusDB
- statusDB
- P: 08-xxxxxx +

{{samples[0].projects.Customer_ID}}


+ +
@@ -56,7 +54,7 @@

Project Summary

Customer ID (CG Project ID) - * + {{samples[0].projects.Customer_ID}} Analysis date @@ -84,18 +82,20 @@

Sample Summary

Biblioteksprep datum Biblioteksmetod Sekvensering datum - SekvenseringsmetodSekvenseringsmetod + Priority(research/standard) {% for sample in samples %} {{sample.Customer_ID_sample}} {{sample.CG_ID_sample}} - * + {{sample.application_tag}} {{samples[0].projects.date_ordered.date()}} * * * * + * {% endfor %} @@ -315,13 +315,13 @@

Resistance

@@ -46,22 +47,18 @@

{{samples[0].projects.Customer_ID}}


-

Project Summary

+

Projekt Sammanfattning

- - - - - - - +
Project ID (CG Project ID) + Projekt ID (CG Projekt ID) {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Customer ID (CG Project ID) + Kund ID {{samples[0].projects.Customer_ID}}
Analysis date{{samples[0].date_analysis.date()}}
Pipeline versionAnalys pipeline version {{build}}
@@ -71,12 +68,12 @@

Project Summary

-

Sample Summary

+

Prov Sammanfattning

- - + + @@ -91,11 +88,11 @@

Sample Summary

+ + - - - + {% endfor %}
Sample IDCG Sample IDProv IDCG Prov ID Applikations tag Ankomstdatum Biblioteksprep datum {{sample.CG_ID_sample}} {{sample.application_tag}} {{samples[0].projects.date_ordered.date()}}{{sample.date_libprep.date()}} *{{sample.date_sequencing.date()}} ****{{sample.priority}}
@@ -109,15 +106,15 @@

Sample Summary

-

Summary of Results

+

Sammanställning av Resultat

- - + + - - + + {% for sample in samples %} @@ -152,22 +149,22 @@

Summary of Results

-

microSALT Results
- Clinical Microbiology -

{{date}}

+

microSALT Resultat
+ Klinisk Mikrobiologi +

Rapport genererad: {{date}}

Sample IDCG Sample IDProv IDCG Prov ID OrganismSequence TypeThresholdsSekvens TypTröskelvärden
-

Overview

+

Översikt

- + - + - + @@ -175,10 +172,10 @@

Overview

{% if sample.organism is not none %} {% else %} - + {% endif %} - + {% if sample.ST_status == 'None' %} {% elif sample.ST <= -4 %} @@ -187,7 +184,7 @@

Overview

{% endif %} - + {% if sample.threshold=='Failed' %} {% else %} @@ -196,7 +193,7 @@

Overview

{% if sample.organism in version %} - + {% endif %} @@ -211,15 +208,15 @@

Overview

Project ID (CG Project ID)Projekt ID (CG Projekt ID) {{sample.projects.Customer_ID_project}} ({{sample.projects.CG_ID_project}})
Sample ID (CG Sample ID)Prov ID (CG Prov ID) {{sample.Customer_ID_sample}} ({{sample.CG_ID_sample}})
Analysis dateAnalys datum {{sample.date_analysis.date()}}
{{sample.organism.replace('_', ' ').capitalize()}}ControlKontroll
Sequence Type
Sekvens Typ{{sample.ST_status }} {{sample.ST_status }}
Thresholds
Tröskelvärde{{sample.threshold }}
Reference versionReferens version {{version[sample.organism]}}

Assembly

{% if sample.genome_length > -1 %} - + - - + + {% else %} - + - - + + {% endif %}
Estimated genome length{{'{0:,}'.format(sample.genome_length)|replace(","," ")}}
Estimerad genom storlek{{'{0:,}'.format(sample.genome_length)|replace(","," ")}}
GC content{{sample.gc_percentage| round(2)}}%
N50 (min. contig length for 50% of genome){{'{0:,}'.format(sample.n50)|replace(","," ")}}
Necessary contigs{{sample.contigs}}
N50 (min. contig längd för 50% av genom){{'{0:,}'.format(sample.n50)|replace(","," ")}}
Nödvändiga contigs{{sample.contigs}}
Estimated genome lengthInsufficient data for QUAST
Estimerad genom storlekInsufficient data for QUAST
GC contentInsufficient data for QUAST
N50 (min. contig length for 50% of genome)Insufficient data for QUAST
Necessary contigsInsufficient data for QUAST
N50 (min. contig längd för 50% av genom)Insufficient data for QUAST
Nödvändiga contigsInsufficient data for QUAST
@@ -240,9 +237,9 @@

MLST

# Loci - Allele - Contig Coverage - Identity + Allel + Contig Täckning + Identitet Span @@ -265,15 +262,15 @@

MLST

{% if not sample.resistances|length == 0 %} -

Resistance

+

Resistens

- - - - - + + + + + @@ -317,7 +314,7 @@

Resistance

Teknisk beskrivning av analysen : {{samples[0].application_tag}}

- Microbial whole genome routine samples, Nextera prep, Sequencing on HiSeq X to 3 M read pairs
+ Mikrobiella helgenoms rutin-prover, Nextera prep, Sekvensering på Illumina till 3 M read par

From 5d36ad791a56cfae4a4ab91ab6ddf31a8ba10a84 Mon Sep 17 00:00:00 2001 From: Emma Sernstad Date: Thu, 28 Feb 2019 14:38:17 +0100 Subject: [PATCH 09/83] Added swedac logo to sample page if priority is standard --- microSALT/server/templates/report_page.html | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index fe2b2b7c..0537091a 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -8,7 +8,7 @@
MicroSALT Logo - Swedac Logo + Swedac Logo



Klinisk Mikrobiologi

Rapport genererad: {{date}}

@@ -151,7 +151,12 @@

Sammanställning av Resultat

microSALT Resultat
Klinisk Mikrobiologi -

Rapport genererad: {{date}}

+

Rapport genererad: {{date}}

 + {% if sample.priority == 'standard' %} + Swedac Logo + {% else %} +

Ej ackrediterad analys

+ {% endif %}
#GeneTypeReferenceContig CoverageIdentityGenTypReferensContig TäckningIdentitet Span

Översikt

From 4d154248ebec22f2cff951ce717e2876515be7f4 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Mon, 18 Mar 2019 11:22:43 +0100 Subject: [PATCH 10/83] Added basic steps table. Still need to rebind all dates to new location --- microSALT/store/db_manipulator.py | 3 +++ microSALT/store/orm_models.py | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index b6564d44..819effce 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -45,6 +45,9 @@ def create_tables(self): if not self.engine.dialect.has_table(self.engine, 'resistances'): Resistances.__table__.create(self.engine) self.logger.info("Created resistance table") + if not self.engine.dialect.has_table(self.engine, 'steps'): + Resistances.__table__.create(self.engine) + self.logger.info("Created step table") for k,v in self.profiles.items(): if not self.engine.dialect.has_table(self.engine, "profile_{}".format(k)): self.profiles[k].create() diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index 7f6eb04e..f9930909 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -23,17 +23,17 @@ class Samples(db.Model): Customer_ID_sample = db.Column(db.String(40)) organism = db.Column(db.String(30)) ST = db.Column(db.SmallInteger, default=-1) - date_analysis = db.Column(db.DateTime) genome_length = db.Column(db.Integer, default=-1) gc_percentage = db.Column(db.Float(3,2), default = 0.0) n50 = db.Column(db.Integer, default=-1) contigs = db.Column(db.Integer, default=-1) priority = db.Column(db.String(20)) application_tag = db.Column(db.String(15)) - date_sequencing = db.Column(db.DateTime) - date_libprep = db.Column(db.DateTime) - method_sequencing = db.Column(db.String(15)) - method_libprep = db.Column(db.String(15)) +# date_analysis = db.Column(db.DateTime) +# date_sequencing = db.Column(db.DateTime) +# date_libprep = db.Column(db.DateTime) +# method_sequencing = db.Column(db.String(15)) +# method_libprep = db.Column(db.String(15)) class Seq_types(db.Model): __tablename__ = 'seq_types' @@ -75,13 +75,20 @@ class Resistances(db.Model): contig_start=db.Column(db.Integer) contig_end=db.Column(db.Integer) +class Steps(db.Model): + __tablename__ = 'steps' + CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) + date = db.Column(db.DateTime) + method = db.Column(db.String(40)) + step = db.Column(db.String(40)) + class Projects(db.Model): __tablename__ = 'projects' samples = relationship('Samples', back_populates='projects') CG_ID_project = db.Column(db.String(15), primary_key=True, nullable=False) Customer_ID_project = db.Column(db.String(15)) - date_ordered = db.Column(db.DateTime) +# date_ordered = db.Column(db.DateTime) Customer_ID = db.Column(db.String(15)) class Versions(db.Model): From 189a79fe1b9ad7ace4ac8556667d63a57b4faa5e Mon Sep 17 00:00:00 2001 From: Emma Sernstad Date: Fri, 22 Mar 2019 10:33:19 +0100 Subject: [PATCH 11/83] Some changes to layout, and text --- microSALT/server/templates/report_page.html | 98 +++++++++++++++------ 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index 0537091a..2ba623ef 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -14,9 +14,29 @@

Rapport genererad: {{date}}

+
+
+

Kontakt Clinical Genomics

+
+ Clinical Genomics
+ Science for Life Laboratory
+ Tomtebodavägen 23
+ 171 65 Solna
+ P: (08) 524 81 500 +
+
+
+

Kund:

+
+

{{samples[0].projects.Customer_ID}}


+
+
+
+
+ +
+

Projekt Sammanfattning(*)

- @@ -64,23 +81,27 @@

Projekt Sammanfattning

Projekt ID (CG Projekt ID) {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Kund ID + Kund ID (*) {{samples[0].projects.Customer_ID}}
+
+
- - + +
+

Prov Sammanfattning(*)

- + - + - + + {% for sample in samples %} @@ -93,26 +114,36 @@

Prov Sammanfattning

+ {% if sample.priority == 'standard' %} + + {% else %} + + {% endif %} {% endfor %} -
Prov IDProv ID (*) CG Prov IDApplikations tagApplikations tag (*) Ankomstdatum Biblioteksprep datum Biblioteksmetod Sekvensering datum SekvenseringsmetodPriority(research/standard)Prioritet (*)(**)Ackrediterad
{{sample.date_sequencing.date()}} * {{sample.priority}}Swedac Logo-
+ +
+

+ (*) Information om prover som kommer från kund
+ (**) Standard (ackrediterad) eller Research (inte ackrediterad)

-
- - - + -
+
-

Sammanställning av Resultat

+

Sammanställning av Resultat(*)

+

+ * Laboratoriet har inte haft ansvar för provtagningsstadiet och + extraktion, resultaten gäller för provet såsom det har mottagits. +

- + - + @@ -143,7 +174,10 @@

Sammanställning av Resultat

- + + + + {% for sample in samples %}
@@ -226,11 +260,10 @@

Assembly

Prov IDProv ID(*) CG Prov IDOrganismOrganism(*) Sekvens Typ Tröskelvärden
-
-
+ {% if sample.genome_length > -1 %} -
+
@@ -329,6 +362,15 @@

Begränsningar av analysen : {{samples[0].application_tag}}

+

Avvikelser från metoden

+

+ All kommunikation gällande ordern såsom tillägg, avvikelser eller + ev undantag i metoden från Clinical Genomics finns tillgängligt i + SupportSystem för dess ticket id. En stängd ticket kan närsomhelst + öppnas upp igen för frågor. +

+ +

Signatur för godkännande av rapport

Valtteri Wirta
From 98afedf7b3b0873a9249907fa04e18a9f7f265d7 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 22 Mar 2019 15:29:09 +0100 Subject: [PATCH 12/83] Nicer report, NTC page split fix, database lookup fixes --- microSALT/server/templates/report_page.html | 99 +++++++++++---------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index 2ba623ef..4cebb9fe 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -64,7 +64,7 @@

{{samples[0].projects.Customer_ID}}



-->
-

Projekt Sammanfattning(*)

+

Projektsammanfattning(*)

- +
Projekt ID (CG Projekt ID) @@ -75,7 +75,7 @@

Projekt Sammanfattning(*)

{{samples[0].projects.Customer_ID}}
Analys pipeline versionVersion av analys pipeline {{build}}
@@ -88,17 +88,17 @@

Projekt Sammanfattning(*)

Prov Sammanfattning

-->
-

Prov Sammanfattning(*)

+

Provsammanfattning(*)

- + - + - + @@ -109,9 +109,17 @@

Prov Sammanfattning(*)

- + {% if sample.date_libprep is not none %} + + {% else %} + + {% endif %} - + {% if sample.date_sequencing is not none %} + + {% else %} + + {% endif %} {% if sample.priority == 'standard' %} @@ -133,7 +141,7 @@

Prov Sammanfattning(*)

-

Sammanställning av Resultat(*)

+

Resultatsammanställning(*)

* Laboratoriet har inte haft ansvar för provtagningsstadiet och extraktion, resultaten gäller för provet såsom det har mottagits. @@ -144,7 +152,7 @@

Sammanställning av Resultat(*)

- + {% for sample in samples %} @@ -154,7 +162,7 @@

Sammanställning av Resultat(*)

{% if sample.organism is not none %} {% else %} - + {% endif %} {% if sample.ST_status == 'None' or sample.ST == -1 %} @@ -164,9 +172,9 @@

Sammanställning av Resultat(*)

{% endif %} {% if sample.threshold == 'Failed' %} - + {% else %} - + {% endif %} {% endfor %} @@ -214,7 +222,7 @@

Översikt

{% endif %} - + {% if sample.ST_status == 'None' %} {% elif sample.ST <= -4 %} @@ -223,16 +231,16 @@

Översikt

{% endif %} - + {% if sample.threshold=='Failed' %} - + {% else %} - + {% endif %} {% if sample.organism in version %} - + {% endif %} @@ -243,22 +251,17 @@

Översikt

{% endif %}
Prov ID (*) CG Prov IDApplikations tag (*)Applikationstag (*) AnkomstdatumBiblioteksprep datumBiblioteksprepsdatum BiblioteksmetodSekvensering datumSekvenseringsdatum Sekvenseringsmetod Prioritet (*)(**) Ackrediterad {{sample.CG_ID_sample}} {{sample.application_tag}} {{samples[0].projects.date_ordered.date()}}{{sample.date_libprep.date()}}{{sample.date_libprep.date()}}0000-00-00*{{sample.date_sequencing.date()}}{{sample.date_sequencing.date()}}0000-00-00* {{sample.priority}} Prov ID(*) CG Prov ID Organism(*)Sekvens TypSekvenstyp Tröskelvärden
{{sample.organism.replace('_', ' ').capitalize()}}ControlKontroll{{ sample.ST_status }} {{sample.ST_status }}{{sample.threshold }}Underkända{{sample.threshold }}Godkända
Kontroll
Sekvens Typ
Sekvenstyp{{sample.ST_status }} {{sample.ST_status }}
Tröskelvärde
Tröskelvärden{{sample.threshold }}Underkända{{sample.threshold }}Godkända
Referens versionVersion av referens {{version[sample.organism]}}
- -
-

Assembly

- {% if sample.genome_length > -1 %} - - - - - {% else %} - - - - - {% endif %} -
Estimerad genom storlek{{'{0:,}'.format(sample.genome_length)|replace(","," ")}}
GC content{{sample.gc_percentage| round(2)}}%
N50 (min. contig längd för 50% av genom){{'{0:,}'.format(sample.n50)|replace(","," ")}}
Nödvändiga contigs{{sample.contigs}}
Estimerad genom storlekInsufficient data for QUAST
GC contentInsufficient data for QUAST
N50 (min. contig längd för 50% av genom)Insufficient data for QUAST
Nödvändiga contigsInsufficient data for QUAST
- + {% if sample.genome_length > -1 %} +
+

Assembly

+ + + + +
Förväntad genomstorlek{{'{0:,}'.format(sample.genome_length)|replace(","," ")}}
GC halt{{sample.gc_percentage| round(2)}}%
N50 (minsta contiglängd för 50% av genomet){{'{0:,}'.format(sample.n50)|replace(","," ")}}
Nödvändiga contigs{{sample.contigs}}
+ {% else %} +
+ {% endif %}
{% if sample.genome_length > -1 %} @@ -278,7 +281,7 @@

MLST

Allel Contig Täckning Identitet - Span + Längd @@ -286,10 +289,10 @@

MLST

{{loop.index}} {{seq_type.loci}} - {{ seq_type.allele}} - {{ seq_type.contig_coverage|round(2) }} - {{ seq_type.identity|round(2) }} - {{ seq_type.span|round(2) }} + {{seq_type.allele}} + {{seq_type.contig_coverage|round(2)}}x + {{seq_type.identity|round(2)}}% + {{(seq_type.span*100)|round(2)}}% {% endfor %} @@ -300,7 +303,7 @@

MLST

{% if not sample.resistances|length == 0 %} -

Resistens

+

Resistenser

@@ -309,7 +312,7 @@

Resistens

- + @@ -327,12 +330,12 @@

Resistens

{% else %} {% endif %} - - + + {% if seq_type.span == 999.0 %} {% else %} - + {% endif %} {% endfor %} @@ -352,14 +355,16 @@

Resistens

Teknisk beskrivning av analysen : {{samples[0].application_tag}}

- Mikrobiella helgenoms rutin-prover, Nextera prep, Sekvensering på Illumina till 3 M read par
+ Mikrobiell helgenomssekvensering av rutinprov. Minst 3 miljoner läspar. Nextera prep.

Begränsningar av analysen : {{samples[0].application_tag}}

-
- + Analysen kan enbart att beställas av gruppen klinisk mikrobiologi. + Typningen utgår från de publikt tillgängliga databaserna pubMLST och resFinder. + Typningen av prover är enbart att anses som tillförlitlig om desamma uppnår de fördefinierade tröskelvärdena. + Uppnåelse av tröskelvärdena garanteras i sin tur enbart för de prover som flaggats som ackrediterade.

Avvikelser från metoden

From b0943894caafebc41296ef5a9740c99610b2cfed Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 22 Mar 2019 15:59:37 +0100 Subject: [PATCH 13/83] Unincluded files from prev. commit --- microSALT/server/views.py | 2 +- microSALT/store/orm_models.py | 12 ++++++------ microSALT/utils/reporter.py | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/microSALT/server/views.py b/microSALT/server/views.py index d4047728..519aa84b 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -79,7 +79,7 @@ def gen_reportdata(pid, organism_group='all'): s.Customer_ID_sample.startswith('NK-') or s.Customer_ID_sample.startswith('NEG') or \ s.Customer_ID_sample.startswith('CTRL') or s.Customer_ID_sample.startswith('Neg') or \ s.Customer_ID_sample.startswith('blank'): - s.ST_status = 'Control (prefix)' + s.ST_status = 'Kontroll (prefix)' elif s.ST < 0: if s.ST == -1: s.ST_status = 'Invalid data' diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index f9930909..d28cef5a 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -29,11 +29,11 @@ class Samples(db.Model): contigs = db.Column(db.Integer, default=-1) priority = db.Column(db.String(20)) application_tag = db.Column(db.String(15)) -# date_analysis = db.Column(db.DateTime) -# date_sequencing = db.Column(db.DateTime) -# date_libprep = db.Column(db.DateTime) -# method_sequencing = db.Column(db.String(15)) -# method_libprep = db.Column(db.String(15)) + date_analysis = db.Column(db.DateTime) + date_sequencing = db.Column(db.DateTime) + date_libprep = db.Column(db.DateTime) + method_sequencing = db.Column(db.String(15)) + method_libprep = db.Column(db.String(15)) class Seq_types(db.Model): __tablename__ = 'seq_types' @@ -88,7 +88,7 @@ class Projects(db.Model): CG_ID_project = db.Column(db.String(15), primary_key=True, nullable=False) Customer_ID_project = db.Column(db.String(15)) -# date_ordered = db.Column(db.DateTime) + date_ordered = db.Column(db.DateTime) Customer_ID = db.Column(db.String(15)) class Versions(db.Model): diff --git a/microSALT/utils/reporter.py b/microSALT/utils/reporter.py index 930cc762..baa189ca 100644 --- a/microSALT/utils/reporter.py +++ b/microSALT/utils/reporter.py @@ -61,6 +61,7 @@ def gen_html(self): self.kill_flask() def mail(self): + """ Requires that the sending PC has a domain""" file_name = self.attachments msg = MIMEMultipart() msg['Subject'] = '{} ({}) Reports'.format(self.name, file_name[0].split('_')[0]) From a55d7b9cbe43354be91e1f6df995cc6da7912b2d Mon Sep 17 00:00:00 2001 From: sylvinite Date: Fri, 22 Mar 2019 16:51:06 +0100 Subject: [PATCH 14/83] swedish characters fix --- microSALT/utils/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/utils/reporter.py b/microSALT/utils/reporter.py index baa189ca..2376f835 100644 --- a/microSALT/utils/reporter.py +++ b/microSALT/utils/reporter.py @@ -56,7 +56,7 @@ def gen_html(self): self.kill_flask() sys.exit(-1) outname = "{}_microSALT.html".format(self.ticketFinder.data['Customer_ID_project']) - open(outname, 'wb').write(r.content) + open(outname, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) self.attachments.append(outname) self.kill_flask() From 060af6422f956c4767a3ebd9844f4a1d6aabd81e Mon Sep 17 00:00:00 2001 From: sylvinite Date: Mon, 25 Mar 2019 09:09:58 +0100 Subject: [PATCH 15/83] Added method retrieval samples --- microSALT/store/lims_fetcher.py | 71 +++++++++++++++++---------------- microSALT/utils/job_creator.py | 2 + 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 40c027cc..66ac5fde 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -63,13 +63,17 @@ def load_lims_sample_info(self, cg_sampleid, external=False): if len(sample) != 1: self.logger.error("Sample ID {} resolves to multiple entries".format(cg_sampleid)) sample = sample[0] - seq_date = self.get_date(sample.id,type="sequencing") - libprep_date = self.get_date(sample.id,type="libprep") + method_libprep = self.get_method(cg_sampleid,type='libprep') + method_sequencing = self.get_method(cg_sampleid,type='sequencing') + date_libprep = self.get_date(cg_sampleid,type="libprep") + date_sequencing = self.get_date(cg_sampleid,type="sequencing") else: try: sample = Sample(self.lims, id=cg_sampleid) - seq_date = self.get_date(cg_sampleid,type="sequencing") - libprep_date = self.get_date(cg_sampleid,type="libprep") + method_libprep = self.get_method(cg_sampleid,type='libprep') + method_sequencing = self.get_method(cg_sampleid,type='sequencing') + date_libprep = self.get_date(cg_sampleid,type="libprep") + date_sequencing = self.get_date(cg_sampleid,type="sequencing") except Exception as e: self.logger.error("LIMS connection timeout") organism = "Unset" @@ -111,8 +115,10 @@ def load_lims_sample_info(self, cg_sampleid, external=False): 'priority' : sample.udf['priority'], 'Customer_ID': sample.udf['customer'], 'application_tag': sample.udf['Sequencing Analysis'], - 'date_sequencing': seq_date, - 'date_libprep': libprep_date + 'date_sequencing': date_sequencing, + 'date_libprep': date_libprep, + 'method_libprep': method_libprep, + 'method_sequencing': method_sequencing }) except KeyError as e: self.logger.warn("Unable to fetch LIMS info for sample {}. Review LIMS data.\nSource: {}"\ @@ -143,21 +149,6 @@ def get_organism_refname(self, sample_name, external=False): self.logger.warn("Unable to find existing reference for {}, strain {} has no reference match\nSource: {}"\ .format(sample_name, lims_organ, e)) -# def get_date(self, sample_id, type=""): -# """ Returns the most recent sequencing date of a sample """ -# if type == "sequencing": -# steps = ["CG002 - Illumina Sequencing (Illumina SBS)", "CG002 Illumina SBS (HiSeq X)"] -# elif type == "libprep": -# steps = "CG002 - Aggregate QC (Library Validation)" -# else: -# raise Exception("Attempted to get date for {} but no step defined".format(sample_id)) -# try: -# arts = self.lims.get_artifacts(samplelimsid = sample_id, process_type = steps) -# date_list = [a.parent_process.date_run for a in arts] -# except Exception as e: -# pass -# return max(date_list) - def get_date(self, sample_id, type=""): """ Returns the most recent sequencing date of a sample """ date_list = list() @@ -176,17 +167,27 @@ def get_date(self, sample_id, type=""): dp = max(date_list).split('-') return datetime(int(dp[0]), int(dp[1]), int(dp[2])) -#DEBUG: Implement later! -# def get_method_document(sample_id, first_date = True): -# steps = ['CG002 - Cluster Generation (HiSeq X)', 'CG002 - Cluster Generation (Illumina SBS)'] -# -# arts = lims.get_artifacts(samplelimsid = sample_id, process_type = steps) -# processes = [(a.parent_process.date_run, a.parent_process.date_run) for a in arts] -# processes = list(set(processes)) -# if first_date -# process = sorted(processes)[0] -# else: -# process = sorted(processes)[-1] -# dokument = process.udf.get('Method Document') -# version = process.udf.get('document version') -# return dokument, version + def get_method(self, sample_id, type=""): + """Retrives method document name and version for a sample""" + key_values = dict() + if type == "libprep": + #MIGHT BE CALLED JUST METHOD AND NOT METHOD DOCUMENT HERE! + steps = ['CG002 - Microbial Library Prep (Nextera)'] + key_values['method'] = "Method" + key_values['version'] = "Method Version" + elif type == "sequencing": + steps = ['CG002 - Cluster Generation (Illumina SBS)', 'CG002 - Cluster Generation (HiSeq X)'] + key_values['method'] = "Method Document 1" + key_values['version'] = "Document 1 Version" + else: + raise Exception("Attempted to get info for {} but no step defined".format(sample_id)) + for step in steps: + try: + arts = self.lims.get_artifacts(samplelimsid = sample_id, process_type = step) + processes = [(a.parent_process.udf[key_values['method']], a.parent_process.udf[key_values['version']]) for a in arts] + processes = list(set(processes)) + process = sorted(processes)[-1] + out = "{}:{}".format(process[0], process[1]) + except Exception as e: + pass + return out diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 8c8286a6..bf80fb9b 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -217,6 +217,8 @@ def create_sample(self, name): sample_col["priority"] = self.lims_fetcher.data['priority'] sample_col["date_sequencing"] = self.lims_fetcher.data['date_sequencing'] sample_col["date_libprep"] = self.lims_fetcher.data['date_libprep'] + sample_col["method_libprep"] = self.lims_fetcher.data['method_libprep'] + sample_col["method_sequencing"] = self.lims_fetcher.data['method_sequencing'] #self.db_pusher.purge_rec(sample_col['CG_ID_sample'], 'sample') self.db_pusher.add_rec(sample_col, 'Samples') except Exception as e: From 9813280e46b1387d280623ffcc124b3d1d454d97 Mon Sep 17 00:00:00 2001 From: Emma Sernstad Date: Mon, 25 Mar 2019 09:11:43 +0100 Subject: [PATCH 16/83] Small changes to text and table --- microSALT/server/templates/report_page.html | 61 ++++++--------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index 4cebb9fe..a7628ffa 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -34,37 +34,10 @@

{{samples[0].projects.Customer_ID}}




- + +
-

Projektsammanfattning(*)

+

Projektsammanställning

# Referens Contig Täckning IdentitetSpanLängd
Unknown{{ seq_type.contig_coverage|round(2) }}{{ seq_type.identity|round(2) }}{{seq_type.contig_coverage|round(2) }}x{{seq_type.identity|round(2) }}%Undef.{{ seq_type.span|round(2) }}{{(seq_type.span*100)|round(2) }}%
+ {% else %} + + {% endif %} + {% if sample.threshold == 'Failed' %} + + {% else %} + + {% endif %} + + {% endfor %} +
Projekt ID (CG Projekt ID) @@ -84,16 +57,14 @@

Projektsammanfattning(*)


- +
-

Provsammanfattning(*)

+

Provsammanställning

- + @@ -101,12 +72,12 @@

Provsammanfattning(*)

- + {% for sample in samples %} - + {% if sample.date_libprep is not none %} @@ -123,7 +94,7 @@

Provsammanfattning(*)

{% if sample.priority == 'standard' %} - + {% else %} {% endif %} @@ -134,16 +105,20 @@

Provsammanfattning(*)

(*) Information om prover som kommer från kund
- (**) Standard (ackrediterad) eller Research (inte ackrediterad) + (**) Standard = Verifierad organim, + Research = Icke verifierad organism
+ Kvalitetskontrollen har verifierats för ett antal bestämda organismer. + För andra organismer kan ej kriterierna för metoden appliceras. +

-

Resultatsammanställning(*)

+

Resultatsammanställning(***)

- * Laboratoriet har inte haft ansvar för provtagningsstadiet och + (***) Laboratoriet har inte haft ansvar för provtagningsstadiet och extraktion, resultaten gäller för provet såsom det har mottagits.

@@ -163,7 +138,7 @@

Resultatsammanställning(*)

{% else %} - {% endif %} + {% endif %} {% if sample.ST_status == 'None' or sample.ST == -1 %} {% elif sample.ST <= -4 %} @@ -197,7 +172,7 @@

microSALT Resultat {% else %} -

Ej ackrediterad analys

+

Icke verifierad organism

{% endif %}

Prov ID (*)CG Prov IDOrganism (*) Applikationstag (*) Ankomstdatum Biblioteksprepsdatum Sekvenseringsdatum Sekvenseringsmetod Prioritet (*)(**)AckrediteradVerifierad
{{sample.Customer_ID_sample}}{{sample.CG_ID_sample}}{{sample.organism.replace('_', ' ').capitalize()}} {{sample.application_tag}} {{samples[0].projects.date_ordered.date()}} * {{sample.priority}}Swedac Logo✔︎- {{sample.organism.replace('_', ' ').capitalize()}}Kontroll{{ sample.ST_status }}
From 3d658b36cc45971bb0ed9f82e4f1f59d746b3ca8 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Mon, 25 Mar 2019 09:23:22 +0100 Subject: [PATCH 17/83] Removed half-done multi-date support --- microSALT/store/db_manipulator.py | 6 +++--- microSALT/store/orm_models.py | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 819effce..5058bded 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -45,9 +45,9 @@ def create_tables(self): if not self.engine.dialect.has_table(self.engine, 'resistances'): Resistances.__table__.create(self.engine) self.logger.info("Created resistance table") - if not self.engine.dialect.has_table(self.engine, 'steps'): - Resistances.__table__.create(self.engine) - self.logger.info("Created step table") +# if not self.engine.dialect.has_table(self.engine, 'steps'): +# Steps.__table__.create(self.engine) +# self.logger.info("Created step table") for k,v in self.profiles.items(): if not self.engine.dialect.has_table(self.engine, "profile_{}".format(k)): self.profiles[k].create() diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index d28cef5a..7306c2ed 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -75,12 +75,13 @@ class Resistances(db.Model): contig_start=db.Column(db.Integer) contig_end=db.Column(db.Integer) -class Steps(db.Model): - __tablename__ = 'steps' - CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) - date = db.Column(db.DateTime) - method = db.Column(db.String(40)) - step = db.Column(db.String(40)) +#Debug: Multi-date support for libprep/sequencing +#class Steps(db.Model): +# __tablename__ = 'steps' +# CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) +# date = db.Column(db.DateTime) +# method = db.Column(db.String(40)) +# step = db.Column(db.String(40)) class Projects(db.Model): __tablename__ = 'projects' From 1757d10d7d4fdf32a1ecd82fd6a893ed95d23b44 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Mon, 25 Mar 2019 09:55:36 +0100 Subject: [PATCH 18/83] Control translation fix --- microSALT/server/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 519aa84b..1a18e082 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -88,7 +88,7 @@ def gen_reportdata(pid, organism_group='all'): else: s.ST_status='None' - if 'Control' in s.ST_status or s.ST == -1: + if 'Kontroll' in s.ST_status or 'Control' in s.ST_status or s.ST == -1: s.threshold = '-' elif s.ST == -3: s.threshold = 'Failed' From 76f696af944582e2553d2f5ddbdb436f07f9bd29 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Mon, 25 Mar 2019 10:16:37 +0100 Subject: [PATCH 19/83] Added methods to report --- microSALT/server/templates/report_page.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index a7628ffa..047294f9 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -85,13 +85,13 @@

Provsammanställning

{% else %} {% endif %} - + {% if sample.date_sequencing is not none %} {% else %} {% endif %} - + {% if sample.priority == 'standard' %} From 9fca4d8d8c36e7c99eee11fd510f66116658f2f6 Mon Sep 17 00:00:00 2001 From: Emma Sernstad Date: Mon, 25 Mar 2019 10:34:13 +0100 Subject: [PATCH 20/83] Small chnages in table headers --- microSALT/server/templates/report_page.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index a7628ffa..4507c78d 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -67,10 +67,10 @@

Provsammanställning

 - - - - + + + + @@ -126,7 +126,7 @@

Resultatsammanställning(***)

 - + From e7cf6313aecdc8236766afbbcb909cbbab7da080 Mon Sep 17 00:00:00 2001 From: Emma Sernstad Date: Thu, 28 Mar 2019 12:51:29 +0100 Subject: [PATCH 21/83] Changed Projekt ID to Ticket ID --- microSALT/server/templates/report_page.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/microSALT/server/templates/report_page.html b/microSALT/server/templates/report_page.html index 8b8f27bc..eb99456c 100644 --- a/microSALT/server/templates/report_page.html +++ b/microSALT/server/templates/report_page.html @@ -40,7 +40,7 @@

{{samples[0].projects.Customer_ID}}


Projektsammanställning

0000-00-00*{{sample.method_libprep}}{{sample.date_sequencing.date()}}0000-00-00*{{sample.method_sequencing}} {{sample.priority}}✔︎Organism (*) Applikationstag (*) AnkomstdatumBiblioteksprepsdatumBiblioteksmetodSekvenseringsdatumSekvenseringsmetodDatum prepMetod prepDatum sekvenseringMetod sekvensering Prioritet (*)(**) Verifierad
Prov ID(*) CG Prov IDOrganism(*)Organism (*) Sekvenstyp Tröskelvärden
- @@ -178,7 +178,7 @@

microSALT Resultat

Projekt ID (CG Projekt ID) + Ticket ID (CG Projekt ID) {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})

Översikt

- + From b785120a3a3079ce744c860669419068c87578f4 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Tue, 18 Jun 2019 16:04:00 +0200 Subject: [PATCH 22/83] Resolved various integration issues --- .../server/templates/alignment_page.html | 34 ++++++++++--------- microSALT/server/templates/typing_page.html | 24 +++++-------- microSALT/utils/reporter.py | 17 +++++++--- 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index a278402b..c0fd9d5c 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -15,11 +15,11 @@

Project Summary

Projekt ID (CG Projekt ID)Ticket ID (CG Projekt ID) {{sample.projects.Customer_ID_project}} ({{sample.projects.CG_ID_project}})
- - +
Project ID (CG Project ID) + Projekt ID (CG Projekt ID) {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Analysis dateAnalysdatum {{samples[0].date_analysis.date()}}
@@ -29,33 +29,35 @@

Sample Summary

- - - + + + - - - - - - - - - + + + + + + + + + + {% for sample in samples %} - + {% if sample.organism is not none %} {% else %}{% endif %} + + + + + + + + + + +
Sample IDCG Sample IDReference GenomeProv IDCG Prov IDApplikationstag OrganismTotal ReadsMapping rate %Duplication Rate %Median Insert SizeAverage coverage%BP > 10x Coverage%BP > 30x Coverage%BP > 50x Coverage%BP > 100x CoverageReferensgenomeAntal ReadsMappningsgrad %Duplikationsgrad %Insert Storlek (Median)Medeltäckning%BP > 10x Täckning%BP > 30x Täckning%BP > 50x Täckning%BP > 100x Täckning
{{ sample.Customer_ID_sample }} {{ sample.CG_ID_sample }}{{ sample.reference_genome }}{{ sample.application_tag }}{{sample.organism.replace('_', ' ').capitalize()}}Undetermined{{ sample.reference_genome }} {% if sample.total_reads is none %} 0 - {% else %} {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} {% endif %} + {% else %} {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{(sample.total_reads/((sample.application_tag[-1:]|float)*10000))|round(2)}}%){% endif %} {% if sample.mapped_rate is none or sample.mapped_rate == 0 %} diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index a538bac4..9adee5d3 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -64,8 +64,8 @@

Provsammanställning

- + @@ -77,28 +77,22 @@

Provsammanställning

{% for sample in samples %} - + {% if sample.date_libprep is not none %} {% else %} - - {% endif %} - {% if sample.ST_status == 'None' or sample.ST == -1 %} - - {% elif sample.ST <= -4 %} - - + {% endif %} {% if sample.date_sequencing is not none %} {% else %} - + {% endif %} - + {% if sample.priority == 'standard' %} {% else %} @@ -172,7 +166,7 @@

Resultatsammanställning(***)

-

microSALT Resultat
+

Detaljresultat
Klinisk Mikrobiologi

Rapport genererad: {{date}}

{% if sample.priority == 'standard' %} @@ -221,13 +215,13 @@

Översikt

{% if sample.organism in version %} - + {% endif %} {% if sample.priority is not none %} - + {% endif %} @@ -293,7 +287,7 @@

Resistenser

- + diff --git a/microSALT/utils/reporter.py b/microSALT/utils/reporter.py index 0a3907fc..4a4f0e45 100644 --- a/microSALT/utils/reporter.py +++ b/microSALT/utils/reporter.py @@ -38,6 +38,7 @@ def __init__(self, config, log, name = "", output = ""): self.server = Process(target=app.run) self.ticketFinder = LIMS_Fetcher(self.config, self.logger) self.attachments = list() + self.filelist = list() self.error = False self.dt = datetime.now() self.now = time.strftime("{}.{}.{}_{}.{}.{}".\ @@ -65,7 +66,7 @@ def report(self, type='default', customer='all'): self.kill_flask() self.mail() if self.output == "" or self.output == os.getcwd(): - for file in self.attachments: + for file in self.filelist: os.remove(file) def gen_STtracker(self, customer="all", silent=False): @@ -77,7 +78,8 @@ def gen_STtracker(self, customer="all", silent=False): self.kill_flask() sys.exit(-1) outname = "{}/ST_updates_{}.html".format(self.output, self.now) - open(outname, 'wb').write(r.content) + open(outname, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) + self.filelist.append(outname) if not silent: self.attachments.append(outname) @@ -91,7 +93,8 @@ def gen_qc(self,silent=False): try: q = requests.get("http://127.0.0.1:5000/microSALT/{}/qc".format(self.name), allow_redirects=True) outqc = "{}/{}_QC_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) - open(outqc, 'wb').write(q.content) + open(outqc, 'wb').write(q.content.decode("iso-8859-1").encode("utf8")) + self.filelist.append(outqc) if not silent: self.attachments.append(outqc) except Exception as e: @@ -108,7 +111,8 @@ def gen_typing(self,silent=False): try: r = requests.get("http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), allow_redirects=True) outtype = "{}/{}_Typing_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) - open(outtype, 'wb').write(r.content) + open(outtype, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) + self.filelist.append(outtype) if not silent: self.attachments.append(outtype) except Exception as e: @@ -170,6 +174,7 @@ def gen_resistence(self, silent=False): excel.write("{}{}\n".format(pref, hits)) excel.close() + self.filelist.append(output) if not silent: self.attachments.append(output) @@ -190,7 +195,8 @@ def gen_json(self, silent=False): report[s.CG_ID_sample]['blast_pubmlst'] = {'sequence_type':s.ST_status, 'thresholds':s.threshold} report[s.CG_ID_sample]['quast_assembly'] = {'estimated_genome_length':s.genome_length, 'gc_percentage':float(s.gc_percentage), 'n50':s.n50, 'necessary_contigs':s.contigs} report[s.CG_ID_sample]['picard_markduplicate'] = {'insert_size':s.insert_size, 'duplication_rate':s.duplication_rate} - report[s.CG_ID_sample]['microsalt_samtools_stats'] = {'total_reads':s.total_reads, 'mapped_rate':s.mapped_rate, 'average_coverage':s.average_coverage, \ + report[s.CG_ID_sample]['microsalt_samtools_stats'] = {'total_reads':s.total_reads, 'mapped_rate':s.mapped_rate, \ + 'average_coverage':s.average_coverage, \ 'coverage_10x':s.coverage_10x, 'coverage_30x':s.coverage_30x, \ 'coverage_50x':s.coverage_50x, 'coverage_100x':s.coverage_100x} @@ -201,6 +207,7 @@ def gen_json(self, silent=False): #json.dumps(report) #Dumps the json directly with open(output, 'w') as outfile: json.dump(report, outfile) + self.filelist.append(output) if not silent: self.attachments.append(output) From e1e681cd805110a827d42425fd1b54a10d3dbcd5 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 19 Jun 2019 13:30:52 +0200 Subject: [PATCH 23/83] Added zebra pattern to tables --- microSALT/server/templates/layout.html | 14 +------------- microSALT/server/templates/typing_page.html | 12 ++---------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/microSALT/server/templates/layout.html b/microSALT/server/templates/layout.html index 038cfaa7..a672af19 100644 --- a/microSALT/server/templates/layout.html +++ b/microSALT/server/templates/layout.html @@ -13,7 +13,7 @@ @@ -80,9 +72,6 @@ - - - - diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 9adee5d3..399bb557 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -12,8 +12,6 @@



Klinisk Mikrobiologi

Rapport genererad: {{date}}

- -

Kontakt Clinical Genomics

@@ -35,7 +33,6 @@

{{samples[0].projects.Customer_ID}}



-

Projektsammanställning

Prov ID (*)Organism (*) Applikationstag (*)Organism (*) Ankomstdatum Datum prep Metod prep
{{sample.Customer_ID_sample}}{{sample.organism.replace('_', ' ').capitalize()}} {{sample.application_tag}}{{sample.organism.replace('_', ' ').capitalize()}} {{samples[0].projects.date_ordered.date()}}{{sample.date_libprep.date()}}Unknown{{ sample.ST_status }}I{{sample.ST|int|abs}} (Novel)0000-00-00Okänd{{sample.method_libprep}}{{sample.date_sequencing.date()}}0000-00-00Okänd{{sample.method_sequencing}}{{sample.priority}}{{sample.priority.capitalize()}}✔︎
Version av referensReferensversion {{version[sample.organism]}}
PriorityPrioritet {{sample.priority}}
Referens Kmer Täckning Identitet %Längd %Längd (HSP) %
@@ -57,7 +54,6 @@

Projektsammanställning


-

Provsammanställning

@@ -303,12 +299,12 @@

Resistenser

{% if seq_type.reference is not none %}
{% else %} - + {% endif %} {% if seq_type.span == 999.0 %} - + {% else %} {% endif %} @@ -350,7 +346,6 @@

Avvikelser från metoden

öppnas upp igen för frågor.

-

Signatur för godkännande av rapport

Valtteri Wirta
@@ -359,12 +354,9 @@

Signatur för godkännande av rapport

Slut på rapport
- - - {% endblock %} From 8003cebc632f7280dc18a053d04cc403999bab8b Mon Sep 17 00:00:00 2001 From: sylvinite Date: Wed, 19 Jun 2019 13:43:57 +0200 Subject: [PATCH 24/83] Added thresholds info --- microSALT/server/templates/alignment_page.html | 6 +++--- microSALT/server/templates/typing_page.html | 6 ++++-- microSALT/server/views.py | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index c0fd9d5c..1c8fb341 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -9,10 +9,10 @@ MicroSALT Logo

-

Quality Control Report +

Kvalitetskontrollsrapport

{{date}}

-

Project Summary

+

Projektsammanställning

{{seq_type.reference}}UnknownOkänd{{seq_type.contig_coverage|round(2) }}x {{seq_type.identity|round(2) }}%Undef.Odefinierad{{(seq_type.span*100)|round(2) }}%
Projekt ID (CG Projekt ID) @@ -25,7 +25,7 @@

Project Summary

-

Sample Summary

+

Provsammanställning

diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 399bb557..20f5a700 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -10,7 +10,7 @@ Swedac Logo

-

Klinisk Mikrobiologi +

Typningsrapport

Rapport genererad: {{date}}

@@ -234,7 +234,7 @@

Assembly

{% endif %}
- + {% if sample.genome_length > -1 %} @@ -245,6 +245,7 @@

Assembly

{% if not sample.seq_types|length == 0 and 'Invalid' not in sample.ST_status%}

MLST

+
Identity threshold: {{threshold.mlst_id}}%, Span threshold: {{(threshold.mlst_span*100)|round(2)}}%, Novel suspicion threshold: {{threshold.mlst_novel_id}}%
@@ -275,6 +276,7 @@

MLST

{% if not sample.resistances|selectattr('threshold', 'equalto', 'Passed')|list|length == 0 %}
#

Resistenser

+
Identity threshold: {{threshold.motif_id}}%, Span threshold: {{(threshold.motif_span*100)|round(2)}}%
diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 84bceb4a..06b264c9 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -56,6 +56,7 @@ def alignment_page(project): samples = sample_info['samples'], date = date.today().isoformat(), version = sample_info['versions'], + threshold = config['threshold'], ) @app.route('/microSALT//typing/') @@ -66,6 +67,7 @@ def typing_page(project, organism_group): samples = sample_info['samples'], date = date.today().isoformat(), version = sample_info['versions'], + threshold = config['threshold'], build = __version__) From 36635f3ef8c736151ad3165cec34b02181fb86e1 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Wed, 19 Jun 2019 14:50:10 +0200 Subject: [PATCH 25/83] Fixed indendation, readded file not found --- microSALT/server/templates/typing_page.html | 372 ++++++++++---------- 1 file changed, 182 insertions(+), 190 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 20f5a700..ede59d3c 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -32,26 +32,23 @@

{{samples[0].projects.Customer_ID}}




-

Projektsammanställning

#
- - - - - - - - - - -
Ticket ID (CG Projekt ID) - {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Kund ID (*) - {{samples[0].projects.Customer_ID}}
Version av analys pipeline{{build}}
-
- - +
Ticket ID (CG Projekt ID) + {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Kund ID (*) + {{samples[0].projects.Customer_ID}}
microSALT version{{build}}
+

@@ -100,152 +97,144 @@

Provsammanställning

- (*) Information om prover som kommer från kund
- (**) Standard = Verifierad organim, - Research = Icke verifierad organism
- Kvalitetskontrollen har verifierats för ett antal bestämda organismer. - För andra organismer kan ej kriterierna för metoden appliceras. - -

- + (*) Information om prover som kommer från kund
+ (**) Standard = Verifierad organim, + Research = Icke verifierad organism
+ Kvalitetskontrollen har verifierats för ett antal bestämda organismer. + För andra organismer kan ej kriterierna för metoden appliceras. + +
-
-

Resultatsammanställning(***)

-

- (***) Laboratoriet har inte haft ansvar för provtagningsstadiet och - extraktion, resultaten gäller för provet såsom det har mottagits. -

-
- - - - - - - - - {% for sample in samples %} - - - - {% if sample.organism is not none %} - - {% else %} - - {% endif %} - {% if sample.ST_status == 'None' or sample.ST == -1 %} - - {% elif sample.ST <= -4 %} - - {% else %} - - {% endif %} - {% if sample.threshold == 'Failed' %} - - {% else %} - - {% endif %} - - {% endfor %} -
Prov ID(*)CG Prov IDOrganism (*)SekvenstypTröskelvärden
{{sample.Customer_ID_sample}}{{sample.CG_ID_sample}}{{sample.organism.replace('_', ' ').capitalize()}}Kontroll{{ sample.ST_status }}{{sample.ST}} (Novel){{sample.ST_status }}UnderkändaGodkända
-
-
-
-
- - - - - {% for sample in samples %} -
-
-
-

Detaljresultat
- Klinisk Mikrobiologi -

Rapport genererad: {{date}}

- {% if sample.priority == 'standard' %} - Swedac Logo - {% else %} -

Icke verifierad organism

- {% endif %} - -
-

Översikt

- - - - - - - - - - - - +
+

Resultatsammanställning(***)

+

+ (***) Laboratoriet har inte haft ansvar för provtagningsstadiet och + extraktion, resultaten gäller för provet såsom det har mottagits. +

+
+
Ticket ID (CG Projekt ID){{sample.projects.Customer_ID_project}} ({{sample.projects.CG_ID_project}})
Prov ID (CG Prov ID){{sample.Customer_ID_sample}} ({{sample.CG_ID_sample}})
Analys datum{{sample.date_analysis.date()}}
+ + + + + + + + {% for sample in samples %} - + + {% if sample.organism is not none %} - {% else %} - + {% else %} + {% endif %} - - - {% if sample.ST_status == 'None' %} - + {% if sample.ST_status == 'None' or sample.ST == -1 %} + {% elif sample.ST <= -4 %} - - {% else %} - - {% endif %} - - - {% if sample.threshold=='Failed' %} - - {% else %} - - {% endif %} - - {% if sample.organism in version %} - - - - - {% endif %} - {% if sample.priority is not none %} - - - - - {% endif %} -
Prov ID(*)CG Prov IDOrganism (*)SekvenstypTröskelvärden
Organism{{sample.Customer_ID_sample}}{{sample.CG_ID_sample}}{{sample.organism.replace('_', ' ').capitalize()}}OkändKontroll
Sekvenstyp{{sample.ST_status }}{{ sample.ST_status }}I{{sample.ST|int|abs}} (Novel){{sample.ST_status }}
TröskelvärdenUnderkändaGodkända
Referensversion{{version[sample.organism]}}
Prioritet{{sample.priority}}
- {% if sample.genome_length > -1 %} -
-

Assembly

- - - - -
Förväntad genomstorlek{{'{0:,}'.format(sample.genome_length)|replace(","," ")}}
GC halt{{sample.gc_percentage| round(2)}}%
N50 (minsta contiglängd för 50% av genomet){{'{0:,}'.format(sample.n50)|replace(","," ")}}
Nödvändiga contigs{{sample.contigs}}
- {% else %} -
- {% endif %} - - - {% if sample.genome_length > -1 %} + 		{{sample.ST}} (Novel){{sample.ST_status }}UnderkändaGodkända
+
+
+ - + + {% for sample in samples %} +
-
+
+

Detaljresultat
+ Klinisk Mikrobiologi +

Rapport genererad: {{date}}

+ {% if sample.priority == 'standard' %} + Swedac Logo + {% else %} +

Icke verifierad organism

+ {% endif %} + +
+

Översikt

+ + + + + + + + + + + + + + + {% if sample.organism is not none %} + + {% else %} + + {% endif %} + + + {% if sample.ST_status == 'None' %} + + {% elif sample.ST <= -4 %} + + {% else %} + + {% endif %} + + + {% if sample.threshold=='Failed' %} + + {% else %} + + {% endif %} + + {% if sample.organism in version %} + + + + + {% endif %} + {% if sample.priority is not none %} + + + + + {% endif %} +
Ticket ID (CG Projekt ID){{sample.projects.Customer_ID_project}} ({{sample.projects.CG_ID_project}})
Prov ID (CG Prov ID){{sample.Customer_ID_sample}} ({{sample.CG_ID_sample}})
Analys datum{{sample.date_analysis.date()}}
Organism{{sample.organism.replace('_', ' ').capitalize()}}Okänd
Sekvenstyp{{sample.ST_status }}I{{sample.ST|int|abs}} (Novel){{sample.ST_status }}
TröskelvärdenUnderkändaGodkända
Referensversion{{version[sample.organism]}}
Prioritet{{sample.priority}}
+
+

Assembly

+ {% if sample.genome_length > -1 %} + + + + +
Förväntad genomstorlek{{'{0:,}'.format(sample.genome_length)|replace(","," ")}}
GC halt{{sample.gc_percentage| round(2)}}%
N50 (minsta contiglängd för 50% av genomet){{'{0:,}'.format(sample.n50)|replace(","," ")}}
Nödvändiga contigs{{sample.contigs}}
+ {% else %} + No assembly data found + {% endif %} +
-
+ {% if sample.genome_length > -1 %} +
+ +

MLST

+
Identity threshold: {{threshold.mlst_id}}%, Span threshold: {{(threshold.mlst_span*100)|round(2)}}%, Novel suspicion threshold: {{threshold.mlst_novel_id}}%
{% if not sample.seq_types|length == 0 and 'Invalid' not in sample.ST_status%} -
-

MLST

-
Identity threshold: {{threshold.mlst_id}}%, Span threshold: {{(threshold.mlst_span*100)|round(2)}}%, Novel suspicion threshold: {{threshold.mlst_novel_id}}%
@@ -258,25 +247,27 @@
Identity threshold: {{threshold.mlst_id}}%, Span threshold: {{(threshold.mls
{% for seq_type in sample.seq_types if seq_type.st_predictor%} - - - - - - - - + + + + + + + + {% endfor %} - +
#
{{loop.index}}{{seq_type.loci}}{{seq_type.allele}}{{seq_type.contig_coverage|round(2)}}x{{seq_type.identity|round(2)}}%{{(seq_type.span*100)|round(2)}}%
{{loop.index}}{{seq_type.loci}}{{seq_type.allele}}{{seq_type.contig_coverage|round(2)}}x{{seq_type.identity|round(2)}}%{{(seq_type.span*100)|round(2)}}%
+ {% else %} + No MLST data found {% endif %} -
+
-
- {% if not sample.resistances|selectattr('threshold', 'equalto', 'Passed')|list|length == 0 %} +

Resistenser

-
Identity threshold: {{threshold.motif_id}}%, Span threshold: {{(threshold.motif_span*100)|round(2)}}%
+
Identity threshold: {{threshold.motif_id}}%, Span threshold: {{(threshold.motif_span*100)|round(2)}}%
+ {% if not sample.resistances|selectattr('threshold', 'equalto', 'Passed')|list|length == 0 %} @@ -290,39 +281,39 @@
Identity threshold: {{threshold.motif_id}}%, Span threshold: {{(threshold.mo
{% for seq_type in sample.resistances if seq_type.threshold == 'Passed' %} - - - - {% if seq_type.resistance is not none %} - - {% else %} - - {% endif %} - {% if seq_type.reference is not none %} - - {% else %} - - {% endif %} - - - {% if seq_type.span == 999.0 %} - - {% else %} - - {% endif %} - + + + + {% if seq_type.resistance is not none %} + + {% else %} + + {% endif %} + {% if seq_type.reference is not none %} + + {% else %} + + {% endif %} + + + {% if seq_type.span == 999.0 %} + + {% else %} + + {% endif %} + {% endfor %}
#
{{loop.index}}{{seq_type.gene}}{{seq_type.resistance|title}}{{seq_type.instance|title}}{{seq_type.reference}}Okänd{{seq_type.contig_coverage|round(2) }}x{{seq_type.identity|round(2) }}%Odefinierad{{(seq_type.span*100)|round(2) }}%
{{loop.index}}{{seq_type.gene}}{{seq_type.resistance|title}}{{seq_type.instance|title}}{{seq_type.reference}}Okänd{{seq_type.contig_coverage|round(2) }}x{{seq_type.identity|round(2) }}%Odefinierad{{(seq_type.span*100)|round(2) }}%
- {% endif %} + {% else %} + No typing data found + {% endif %} +
+ {% endif %}
- -
- {% endif %} {% endfor %} -
@@ -359,6 +350,7 @@

Signatur för godkännande av rapport

+
{% endblock %} From 94f7f1302385dbc59bcea23bdae473c61f5c5f40 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Wed, 19 Jun 2019 15:01:39 +0200 Subject: [PATCH 26/83] Fixed kontroll issue --- microSALT/server/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 06b264c9..b57f9bce 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -135,7 +135,7 @@ def gen_reportdata(pid='all', organism_group='all'): else: s.threshold = 'Failed' - if not 'Control' in s.ST_status and s.ST < 0: + if not ('Control' in s.ST_status or 'Kontroll' in s.ST_status) and s.ST < 0: if s.ST == -1: s.ST_status = 'Invalid data' elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Passed': From 937bb9844a1ffa3ec8f9e7d4c74bee44e76ce29c Mon Sep 17 00:00:00 2001 From: sylvinite Date: Wed, 19 Jun 2019 19:27:15 +0200 Subject: [PATCH 27/83] Further niceness to reports --- .../server/templates/alignment_page.html | 78 +++++-- microSALT/server/templates/typing_page.html | 221 +++++++++--------- 2 files changed, 171 insertions(+), 128 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 1c8fb341..31746127 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -6,24 +6,53 @@
- MicroSALT Logo + + MicroSALT Logo

-

Kvalitetskontrollsrapport -

{{date}}

+


Kvalitetskontrollsrapport -

Projektsammanställning

-
- - - - - - - -
Projekt ID (CG Projekt ID) - {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Analysdatum{{samples[0].date_analysis.date()}}
+

Rapport genererad: {{date}}

 +
+
+

Kontakt Clinical Genomics

+
+ Clinical Genomics
+ Science for Life Laboratory
+ Tomtebodavägen 23
+ 171 65 Solna
+ P: (08) 524 81 500 +
+
+

Kund:

+
+

{{samples[0].projects.Customer_ID}}


+
+
+
+
+ +
+

Projektsammanställning

+
+ + + + + + + + + + +
Ticket ID (CG Projekt ID) + {{samples[0].projects.Customer_ID_project}} ({{samples[0].projects.CG_ID_project}})
Kund ID (*) + {{samples[0].projects.Customer_ID}}
microSALT version{{build}}
+
+

Provsammanställning

@@ -33,7 +62,7 @@

Provsammanställning

CG Prov ID Applikationstag Organism - Referensgenome + Referensgenom Antal Reads Mappningsgrad % Duplikationsgrad % @@ -105,6 +134,25 @@

Provsammanställning

{% endfor %}
+

Begränsningar av analysen : {{samples[0].application_tag}}

+

+ - +

+ +

Avvikelser från metoden

+

+ All kommunikation gällande ordern såsom tillägg, avvikelser eller + ev undantag i metoden från Clinical Genomics finns tillgängligt i + SupportSystem för dess ticket id. En stängd ticket kan närsomhelst + öppnas upp igen för frågor. +

+ +

Signatur för godkännande av rapport

+

+ Valtteri Wirta
+ Head of unit, Clinical Genomics +

+
Slut på rapport
diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index ede59d3c..35a1b87f 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -6,11 +6,14 @@
- MicroSALT Logo + + MicroSALT Logo Swedac Logo

-

Typningsrapport +


Typningsrapport

Rapport genererad: {{date}}

@@ -49,49 +52,48 @@

Projektsammanställning

-

Provsammanställning

- - - - - - - - - - - - - {% for sample in samples %} - - - - - - {% if sample.date_libprep is not none %} - - {% else %} - - {% endif %} - - {% if sample.date_sequencing is not none %} - - {% else %} - - {% endif %} - - - {% if sample.priority == 'standard' %} - - {% else %} - - {% endif %} + + + + + + + + + + + + {% for sample in samples %} + + + + + + {% if sample.date_libprep is not none %} + + {% else %} + + {% endif %} + + {% if sample.date_sequencing is not none %} + + {% else %} + + {% endif %} + + + {% if sample.priority == 'standard' %} + + {% else %} + + {% endif %} + {% endfor %}
Prov ID (*)Applikationstag (*)Organism (*)AnkomstdatumDatum prepMetod prepDatum sekvenseringMetod sekvenseringPrioritet (*)(**)Verifierad
{{sample.Customer_ID_sample}}{{sample.application_tag}}{{sample.organism.replace('_', ' ').capitalize()}}{{samples[0].projects.date_ordered.date()}}{{sample.date_libprep.date()}}Okänd{{sample.method_libprep}}{{sample.date_sequencing.date()}}Okänd{{sample.method_sequencing}}{{sample.priority.capitalize()}}✔︎-
Prov ID (*)Applikationstag (*)Organism (*)AnkomstdatumDatum prepMetod prepDatum sekvenseringMetod sekvenseringPrioritet (*)(**)Verifierad
{{sample.Customer_ID_sample}}{{sample.application_tag}}{{sample.organism.replace('_', ' ').capitalize()}}{{samples[0].projects.date_ordered.date()}}{{sample.date_libprep.date()}}Okänd{{sample.method_libprep}}{{sample.date_sequencing.date()}}Okänd{{sample.method_sequencing}}{{sample.priority.capitalize()}}✔︎-
@@ -150,6 +152,44 @@

Resultatsammanställning(***)

+
+
+ +

Teknisk beskrivning av analysen : {{samples[0].application_tag}}

+

+ Mikrobiell helgenomssekvensering av rutinprov, med krav på minst 3 miljoner läspar. + Nextera library preparation. +

+ +

Begränsningar av analysen : {{samples[0].application_tag}}

+

+ Analysen kan enbart att beställas av gruppen Klinisk Mikrobiologi. + Typningen utgår från de publikt tillgängliga databaserna pubMLST och resFinder. + Typningen av prover är enbart att anses som tillförlitlig om desamma uppnår de fördefinierade tröskelvärdena. + Uppnåelse av tröskelvärdena garanteras i sin tur enbart för prover som vars förhantering och detaljer tidigare verifierats av personal på Clinical Genomics. +

+ +

Avvikelser från metoden

+

+ All kommunikation gällande order såsom tillägg, avvikelser eller + ev undantag i metoden från Clinical Genomics finns tillgängligt i + SupportSystem för dess ticket id. En stängd ticket kan närsomhelst + öppnas upp igen för frågor. +

+ +

Signatur för godkännande av rapport

+

+ Valtteri Wirta
+ Head of unit, Clinical Genomics +

+ +
Slut på rapport
+ +
+
+
+ + {% for sample in samples %}
@@ -157,14 +197,13 @@

Resultatsammanställning(***)

Detaljresultat
Klinisk Mikrobiologi -

Rapport genererad: {{date}}

{% if sample.priority == 'standard' %} Swedac Logo {% else %}

Icke verifierad organism

{% endif %} - +

Rapport genererad: {{date}}

Översikt

@@ -212,7 +251,7 @@

Översikt

{% if sample.priority is not none %} - + {% endif %}
Prioritet{{sample.priority}}{{sample.priority.capitalize()}}
@@ -225,61 +264,54 @@

Assembly

Nödvändiga contigs{{sample.contigs}}
{% else %} - No assembly data found + Assemblydata saknas {% endif %} {% if sample.genome_length > -1 %} -
- +

MLST

-
Identity threshold: {{threshold.mlst_id}}%, Span threshold: {{(threshold.mlst_span*100)|round(2)}}%, Novel suspicion threshold: {{threshold.mlst_novel_id}}%
+
Identitetströskel: {{threshold.mlst_id}}%, Längdtröskel: {{(threshold.mlst_span*100)|round(2)}}%, Novel misstankströskel: {{threshold.mlst_novel_id}}%
+
{% if not sample.seq_types|length == 0 and 'Invalid' not in sample.ST_status%} - - - - - - - - - - - - {% for seq_type in sample.seq_types if seq_type.st_predictor%} - - - - - - - - - {% endfor %} - -
#LociAllelKmer TäckningIdentitet %Längd %
{{loop.index}}{{seq_type.loci}}{{seq_type.allele}}{{seq_type.contig_coverage|round(2)}}x{{seq_type.identity|round(2)}}%{{(seq_type.span*100)|round(2)}}%
+ + # + Loci + Allel + Kmer Täckning + Identitet % + Längd % + + {% for seq_type in sample.seq_types if seq_type.st_predictor%} + + {{loop.index}} + {{seq_type.loci}} + {{seq_type.allele}} + {{seq_type.contig_coverage|round(2)}}x + {{seq_type.identity|round(2)}}% + {{(seq_type.span*100)|round(2)}}% + + {% endfor %} + {% else %} - No MLST data found + MLST data saknas {% endif %}

Resistenser

-
Identity threshold: {{threshold.motif_id}}%, Span threshold: {{(threshold.motif_span*100)|round(2)}}%
+
Identitetströskel: {{threshold.motif_id}}%, Längdtröskel: {{(threshold.motif_span*100)|round(2)}}%
{% if not sample.resistances|selectattr('threshold', 'equalto', 'Passed')|list|length == 0 %} - - + - - {% for seq_type in sample.resistances if seq_type.threshold == 'Passed' %} @@ -303,10 +335,9 @@
Identity threshold: {{threshold.motif_id}}%, Span threshold: {{(threshold.mo {% endif %}
{% endfor %} -
# GenTypGrupp Referens Kmer Täckning Identitet % Längd (HSP) %
{{loop.index}}
{% else %} - No typing data found + Resistensdata saknas {% endif %}
{% endif %} @@ -314,42 +345,6 @@
Identity threshold: {{threshold.motif_id}}%, Span threshold: {{(threshold.mo
{% endfor %} -
-
- -

Teknisk beskrivning av analysen : {{samples[0].application_tag}}

-

- Mikrobiell helgenomssekvensering av rutinprov. Minst 3 miljoner läspar. Nextera prep.
- -

- -

Begränsningar av analysen : {{samples[0].application_tag}}

-

- Analysen kan enbart att beställas av gruppen klinisk mikrobiologi. - Typningen utgår från de publikt tillgängliga databaserna pubMLST och resFinder. - Typningen av prover är enbart att anses som tillförlitlig om desamma uppnår de fördefinierade tröskelvärdena. - Uppnåelse av tröskelvärdena garanteras i sin tur enbart för de prover som flaggats som ackrediterade. -

- -

Avvikelser från metoden

-

- All kommunikation gällande ordern såsom tillägg, avvikelser eller - ev undantag i metoden från Clinical Genomics finns tillgängligt i - SupportSystem för dess ticket id. En stängd ticket kan närsomhelst - öppnas upp igen för frågor. -

- -

Signatur för godkännande av rapport

-

- Valtteri Wirta
- Head of unit, Clinical Genomics -

- -
Slut på rapport
- -
-
-
From 4e40202ed19375a456168268f342739d2d26d876 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Tue, 25 Jun 2019 15:57:50 +0200 Subject: [PATCH 28/83] More visual fixes --- microSALT/server/templates/alignment_page.html | 15 +++++++++------ microSALT/server/templates/typing_page.html | 16 ++++++++-------- microSALT/server/views.py | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 31746127..624b45f6 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -52,8 +52,7 @@

Projektsammanställning

-
- +

Provsammanställning

@@ -134,12 +133,15 @@

Provsammanställning

{% endfor %}
-

Begränsningar av analysen : {{samples[0].application_tag}}

+
+ +
+

Analysbegränsningar för: {{samples[0].application_tag}}

-

-

Avvikelser från metoden

+

Avvikelser från metoden

All kommunikation gällande ordern såsom tillägg, avvikelser eller ev undantag i metoden från Clinical Genomics finns tillgängligt i @@ -147,13 +149,14 @@

Avvikelser från metoden

öppnas upp igen för frågor.

-

Signatur för godkännande av rapport

+

Signatur för godkännande av rapport

Valtteri Wirta
Head of unit, Clinical Genomics

-
Slut på rapport
+
+
Slut på rapport
diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 35a1b87f..7d91bd14 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -97,13 +97,12 @@

Provsammanställning

{% endfor %} - -

(*) Information om prover som kommer från kund
(**) Standard = Verifierad organim, Research = Icke verifierad organism
Kvalitetskontrollen har verifierats för ett antal bestämda organismer. För andra organismer kan ej kriterierna för metoden appliceras. + @@ -183,7 +182,6 @@

Signatur för godkännande av rapport

Head of unit, Clinical Genomics

-
Slut på rapport
@@ -195,14 +193,15 @@

Signatur för godkännande av rapport

-

Detaljresultat
- Klinisk Mikrobiologi +

Detaljresultat {% if sample.priority == 'standard' %} Swedac Logo {% else %}

Icke verifierad organism

{% endif %} +
+ Klinisk Mikrobiologi

Rapport genererad: {{date}}

Översikt

@@ -262,10 +261,10 @@

Assembly

-
GC halt{{sample.gc_percentage| round(2)}}%
N50 (minsta contiglängd för 50% av genomet){{'{0:,}'.format(sample.n50)|replace(","," ")}}
Nödvändiga contigs{{sample.contigs}}
{% else %} Assemblydata saknas {% endif %} +
{% if sample.genome_length > -1 %} @@ -280,7 +279,7 @@
Identitetströskel: {{threshold.mlst_id}}%, Längdtröskel: {{(threshold.mls Allel Kmer Täckning Identitet % - Längd % + Längd (HSP) % {% for seq_type in sample.seq_types if seq_type.st_predictor%} @@ -294,6 +293,7 @@
Identitetströskel: {{threshold.mlst_id}}%, Längdtröskel: {{(threshold.mls {% endfor %} {% else %} + MLST data saknas {% endif %} @@ -345,7 +345,7 @@
Identitetströskel: {{threshold.motif_id}}%, Längdtröskel: {{(threshold.mo
{% endfor %} - +
Slut på rapport
{% endblock %} diff --git a/microSALT/server/views.py b/microSALT/server/views.py index b57f9bce..f7a1369f 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -57,7 +57,7 @@ def alignment_page(project): date = date.today().isoformat(), version = sample_info['versions'], threshold = config['threshold'], - ) + build = __version__) @app.route('/microSALT//typing/') def typing_page(project, organism_group): From 25e46135aa77780b50311b99ebecc35ca12ea3d0 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Tue, 25 Jun 2019 16:22:36 +0200 Subject: [PATCH 29/83] Language fixes --- microSALT/server/templates/typing_page.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 7d91bd14..e22d063d 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -162,10 +162,10 @@

Teknisk beskrivning av analysen : {{samples[0].application_tag}}Begränsningar av analysen : {{samples[0].application_tag}}

- Analysen kan enbart att beställas av gruppen Klinisk Mikrobiologi. + Analysen kan enbart beställas av gruppen Klinisk Mikrobiologi. Typningen utgår från de publikt tillgängliga databaserna pubMLST och resFinder. Typningen av prover är enbart att anses som tillförlitlig om desamma uppnår de fördefinierade tröskelvärdena. - Uppnåelse av tröskelvärdena garanteras i sin tur enbart för prover som vars förhantering och detaljer tidigare verifierats av personal på Clinical Genomics. + Uppnåelse av tröskelvärdena garanteras i sin tur enbart för prover vars förhantering och detaljer tidigare verifierats av personal på Clinical Genomics.

Avvikelser från metoden

From 92a6e81e6c9c2d2469ec27be7e520e7263217301 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Wed, 26 Jun 2019 08:30:13 +0200 Subject: [PATCH 30/83] Minor clarification QC page --- microSALT/server/templates/alignment_page.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 624b45f6..7e0ca967 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -57,11 +57,11 @@

Provsammanställning

- + - - + + @@ -133,6 +133,7 @@

Provsammanställning

{% endfor %}
Prov IDProv ID (*) CG Prov ID ApplikationstagOrganismReferensgenomOrganism (*)Referensgenom (*) Antal Reads Mappningsgrad % Duplikationsgrad %
+ (*) Information om prover som kommer från kund
From 4671c9cc2e62395d9f7e73d2abd7a397f8e204fe Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 26 Jun 2019 09:12:34 +0200 Subject: [PATCH 31/83] Reintegrated qc-thresholds, db fix, span fix; to new report base --- configExample.json | 23 +++++++++- microSALT/cli.py | 7 ++++ .../server/templates/alignment_page.html | 42 ++++++++++++++++++- microSALT/server/views.py | 6 +-- microSALT/store/db_manipulator.py | 2 +- 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/configExample.json b/configExample.json index d51e724e..219d5d66 100644 --- a/configExample.json +++ b/configExample.json @@ -45,11 +45,30 @@ "_comment": "Thresholds for Displayed results", "threshold": { + "_comment": "Typing thresholds", "mlst_id": 100, "mlst_novel_id": 99.5, - "mlst_span": 0.9, + "mlst_span": 90, "motif_id": 97, - "motif_span": 0.9 + "motif_span": 90, + "_comment": "Quality Control thresholds", + "total_reads_warn": 75, + "total_reads_fail": 70, + "NTC_total_reads_warn": 10, + "NTC_total_reads_fail": 20, + "mapped_rate_warn": 50, + "mapped_rate_fail": 30, + "duplication_rate_warn": 20, + "duplication_rate_fail": 80, + "insert_size_warn": 140, + "insert_size_fail": 100, + "average_coverage_warn": 100, + "average_coverage_fail": 10, + "bp_10x_warn": 90, + "bp_10x_fail": 80, + "bp_30x_warn": 70, + "bp_50x_warn": 50, + "bp_100x_warn": 20 }, "_comment": "Genologics temporary configuration file", diff --git a/microSALT/cli.py b/microSALT/cli.py index bc1aac2e..8a68ca2d 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -7,6 +7,8 @@ import logging import json import os +import re +import subprocess import sys import yaml @@ -22,6 +24,11 @@ if config == '': click.echo("ERROR: No properly set-up config under neither envvar MICROSALT_CONFIG nor ~/.microSALT/config.json. Exiting.") sys.exit(-1) +else: + #Makes sure DB inherits correct permissions if freshly created + bash_cmd="touch {}".format(re.search('sqlite\:\/\/\/(.+)', config['database']['SQLALCHEMY_DATABASE_URI']).group(0)) + subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) + output, error = samproc.communicate() def done(): click.echo("Execution finished!") diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 7e0ca967..b77d073c 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -88,13 +88,23 @@

Provsammanställning

{% else %} {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{(sample.total_reads/((sample.application_tag[-1:]|float)*10000))|round(2)}}%){% endif %} - {% if sample.mapped_rate is none or sample.mapped_rate == 0 %} + {% if sample.mapped_rate is none or sample.mapped_rate == 0 %} 0.0 + {% elif (sample.mapped_rate*100)|round(2) <= {{threshold.mapped_rate_warn }} %} + {% if (sample.mapped_rate*100)|round(2) <= {{threshold.mapped_rate_fail }} %} + {{ (sample.mapped_rate*100)|round(2) }} + {% else %} + {{ (sample.mapped_rate*100)|round(2) }} {% else %} {{ (sample.mapped_rate*100)|round(2) }} {% endif %} {% if sample.duplication_rate is none or sample.duplication_rate == -1 %} N/A + {% elif (sample.duplication_rate*100)|round(2) <= {{threshold.duplication_rate_warn }} %} + {% if (sample.duplication_rate*100)|round(2) <= {{threshold.duplication_rate_fail }} %} + {{ (sample.duplication_rate*100)|round(2) }} + {% else %} + {{ (sample.duplication_rate*100)|round(2) }} {% else %} {{ (sample.duplication_rate*100)|round(2) }} {% endif %} @@ -102,31 +112,61 @@

Provsammanställning

{% if sample.insert_size is none or sample.insert_size == 0%} N/A + {% elif (sample.insert_size <= {{threshold.insert_size_warn }} %} + {% if (sample.insert_size <= {{threshold.insert_size_fail }} %} + {{ insert_size }} + {% else %} + {{ insert_size }} {% else %} {{ sample.insert_size }} {% endif %} {% if sample.average_coverage is none or sample.average_coverage|round(2) == 0 %} 0.0x + {% elif (sample.average_coverage)|round(2) <= {{threshold.average_coverage_warn }} %} + {% if (sample.average_coverage)|round(2) <= {{threshold.average_coverage_fail }} %} + {{ (sample.average_coverage)|round(2) }} + {% else %} + {{ (sample.average_coverage)|round(2) }} {% else %} {{ (sample.average_coverage)|round(2) }}x {% endif %} {% if sample.coverage_10x is none or (sample.coverage_10x*100)|round(2) == 0 %} 0.0 + {% elif (sample.coverage_10x*100)|round(2) <= {{threshold.coverage_10x_warn }} %} + {% if (sample.coverage_10x*100)|round(2) <= {{threshold.coverage_10x_fail }} %} + {{ (sample.coverage_10x*100)|round(2) }} + {% else %} + {{ (sample.coverage_10x*100)|round(2) }} {% else %} {{ (sample.coverage_10x*100)|round(2) }} {% endif %} {% if sample.coverage_30x is none or (sample.coverage_30x*100)|round(2) == 0 %} 0.0 + {% elif (sample.coverage_30x*100)|round(2) <= {{threshold.coverage_30x_warn }} %} + {% if (sample.coverage_30x*100)|round(2) <= {{threshold.coverage_30x_fail }} %} + {{ (sample.coverage_30x*100)|round(2) }} + {% else %} + {{ (sample.coverage_30x*100)|round(2) }} {% else %} {{ (sample.coverage_30x*100)|round(2) }} {% endif %} {% if sample.coverage_50x is none or (sample.coverage_50x*100)|round(2) == 0 %} 0.0 + {% elif (sample.coverage_50x*100)|round(2) <= {{threshold.coverage_50x_warn }} %} + {% if (sample.coverage_50x*100)|round(2) <= {{threshold.coverage_50x_fail }} %} + {{ (sample.coverage_50x*100)|round(2) }} + {% else %} + {{ (sample.coverage_50x*100)|round(2) }} {% else %} {{ (sample.coverage_50x*100)|round(2) }} {% endif %} {% if sample.coverage_100x is none or (sample.coverage_100x*100)|round(2) == 0 %} 0.0 + {% elif (sample.coverage_100x*100)|round(2) <= {{threshold.coverage_100x_warn }} %} + {% if (sample.coverage_100x*100)|round(2) <= {{threshold.coverage_100x_fail }} %} + {{ (sample.coverage_100x*100)|round(2) }} + {% else %} + {{ (sample.coverage_100x*100)|round(2) }} {% else %} {{ (sample.coverage_100x*100)|round(2) }} {% endif %} diff --git a/microSALT/server/views.py b/microSALT/server/views.py index f7a1369f..9b6a04e3 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -124,10 +124,10 @@ def gen_reportdata(pid='all', organism_group='all'): for seq_type in s.seq_types: #Identify single deviating allele if seq_type.st_predictor and seq_type.identity >= config["threshold"]["mlst_novel_id"] and \ - config["threshold"]["mlst_id"] > seq_type.identity and 1-abs(1-seq_type.span) >= config["threshold"]["mlst_span"]: + config["threshold"]["mlst_id"] > seq_type.identity and 1-abs(1-seq_type.span) >= (config["threshold"]["mlst_span"]/100.0): near_hits = near_hits + 1 elif (seq_type.identity < config["threshold"]["mlst_novel_id"] or \ - seq_type.span < config["threshold"]["mlst_span"]) and seq_type.st_predictor: + seq_type.span < (config["threshold"]["mlst_span"]/100.0)) and seq_type.st_predictor: s.threshold = 'Failed' if near_hits > 0 and s.threshold == 'Passed': @@ -148,7 +148,7 @@ def gen_reportdata(pid='all', organism_group='all'): #Resistence filter for r in s.resistances: if (s.ST > 0 or 'Novel' in s.ST_status ) and (r.identity >= config["threshold"]["motif_id"] and \ - r.span >= config["threshold"]["motif_span"]) or (s.ST < 0 and not 'Novel' in s.ST_status): + r.span >= (config["threshold"]["motif_span"]/100.0)) or (s.ST < 0 and not 'Novel' in s.ST_status): r.threshold = 'Passed' else: r.threshold = 'Failed' diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 16c4ba4a..af206a6b 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -507,7 +507,7 @@ def bestAlleles(self, cg_sid): def get_unique_alleles(self, cg_sid, organism, threshold=True): """ Returns a dict containing all unique alleles at every loci, and allele difference from expected""" tid = float(self.config["threshold"]["mlst_id"]) - tspan = float(self.config["threshold"]["mlst_span"]) + tspan = (self.config["threshold"]["mlst_span"])/100.0 if threshold: hits = self.session.query(Seq_types.loci, Seq_types.allele)\ .filter(Seq_types.CG_ID_sample==cg_sid, Seq_types.identity >= tid, Seq_types.span >= tspan).all() From 0de2490c4f4ca1868a2f5d5937d092593857fae8 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 26 Jun 2019 10:35:46 +0200 Subject: [PATCH 32/83] Various fixes --- microSALT/cli.py | 6 +-- .../server/templates/alignment_page.html | 44 +++++++++---------- microSALT/server/templates/typing_page.html | 4 +- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/microSALT/cli.py b/microSALT/cli.py index 8a68ca2d..fc7f77ad 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -26,9 +26,9 @@ sys.exit(-1) else: #Makes sure DB inherits correct permissions if freshly created - bash_cmd="touch {}".format(re.search('sqlite\:\/\/\/(.+)', config['database']['SQLALCHEMY_DATABASE_URI']).group(0)) - subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) - output, error = samproc.communicate() + bash_cmd="touch {}".format(re.search('sqlite\:\/\/\/(.+)', config['database']['SQLALCHEMY_DATABASE_URI']).group(1)) + proc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) + output, error = proc.communicate() def done(): click.echo("Execution finished!") diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index b77d073c..ce71f174 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -88,23 +88,25 @@

Provsammanställning

{% else %} {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{(sample.total_reads/((sample.application_tag[-1:]|float)*10000))|round(2)}}%){% endif %} - {% if sample.mapped_rate is none or sample.mapped_rate == 0 %} - 0.0 - {% elif (sample.mapped_rate*100)|round(2) <= {{threshold.mapped_rate_warn }} %} - {% if (sample.mapped_rate*100)|round(2) <= {{threshold.mapped_rate_fail }} %} + {% if sample.mapped_rate is none or sample.mapped_rate == 0 %} + 0.0 + {% elif (sample.mapped_rate*100)|round(2) <= threshold.mapped_rate_warn %} + {% if (sample.mapped_rate*100)|round(2) <= threshold.mapped_rate_fail %} {{ (sample.mapped_rate*100)|round(2) }} {% else %} {{ (sample.mapped_rate*100)|round(2) }} + {% endif %} {% else %} {{ (sample.mapped_rate*100)|round(2) }} {% endif %} {% if sample.duplication_rate is none or sample.duplication_rate == -1 %} N/A - {% elif (sample.duplication_rate*100)|round(2) <= {{threshold.duplication_rate_warn }} %} - {% if (sample.duplication_rate*100)|round(2) <= {{threshold.duplication_rate_fail }} %} + {% elif (sample.duplication_rate*100)|round(2) <= threshold.duplication_rate_warn %} + {% if (sample.duplication_rate*100)|round(2) <= threshold.duplication_rate_fail %} {{ (sample.duplication_rate*100)|round(2) }} {% else %} {{ (sample.duplication_rate*100)|round(2) }} + {% endif %} {% else %} {{ (sample.duplication_rate*100)|round(2) }} {% endif %} @@ -112,60 +114,54 @@

Provsammanställning

{% if sample.insert_size is none or sample.insert_size == 0%} N/A - {% elif (sample.insert_size <= {{threshold.insert_size_warn }} %} - {% if (sample.insert_size <= {{threshold.insert_size_fail }} %} + {% elif sample.insert_size <= threshold.insert_size_warn %} + {% if sample.insert_size <= threshold.insert_size_fail %} {{ insert_size }} {% else %} {{ insert_size }} + {% endif %} {% else %} {{ sample.insert_size }} {% endif %} {% if sample.average_coverage is none or sample.average_coverage|round(2) == 0 %} 0.0x - {% elif (sample.average_coverage)|round(2) <= {{threshold.average_coverage_warn }} %} - {% if (sample.average_coverage)|round(2) <= {{threshold.average_coverage_fail }} %} + {% elif (sample.average_coverage)|round(2) <= threshold.average_coverage_warn %} + {% if (sample.average_coverage)|round(2) <= threshold.average_coverage_fail %} {{ (sample.average_coverage)|round(2) }} {% else %} {{ (sample.average_coverage)|round(2) }} + {% endif %} {% else %} {{ (sample.average_coverage)|round(2) }}x {% endif %} {% if sample.coverage_10x is none or (sample.coverage_10x*100)|round(2) == 0 %} 0.0 - {% elif (sample.coverage_10x*100)|round(2) <= {{threshold.coverage_10x_warn }} %} - {% if (sample.coverage_10x*100)|round(2) <= {{threshold.coverage_10x_fail }} %} + {% elif (sample.coverage_10x*100)|round(2) <= threshold.bp_10x_warn %} + {% if (sample.coverage_10x*100)|round(2) <= threshold.bp_10x_fail %} {{ (sample.coverage_10x*100)|round(2) }} {% else %} {{ (sample.coverage_10x*100)|round(2) }} + {% endif %} {% else %} {{ (sample.coverage_10x*100)|round(2) }} {% endif %} {% if sample.coverage_30x is none or (sample.coverage_30x*100)|round(2) == 0 %} 0.0 - {% elif (sample.coverage_30x*100)|round(2) <= {{threshold.coverage_30x_warn }} %} - {% if (sample.coverage_30x*100)|round(2) <= {{threshold.coverage_30x_fail }} %} - {{ (sample.coverage_30x*100)|round(2) }} - {% else %} + {% elif (sample.coverage_30x*100)|round(2) <= threshold.bp_30x_warn %} {{ (sample.coverage_30x*100)|round(2) }} {% else %} {{ (sample.coverage_30x*100)|round(2) }} {% endif %} {% if sample.coverage_50x is none or (sample.coverage_50x*100)|round(2) == 0 %} 0.0 - {% elif (sample.coverage_50x*100)|round(2) <= {{threshold.coverage_50x_warn }} %} - {% if (sample.coverage_50x*100)|round(2) <= {{threshold.coverage_50x_fail }} %} - {{ (sample.coverage_50x*100)|round(2) }} - {% else %} + {% elif (sample.coverage_50x*100)|round(2) <= threshold.bp_50x_warn %} {{ (sample.coverage_50x*100)|round(2) }} {% else %} {{ (sample.coverage_50x*100)|round(2) }} {% endif %} {% if sample.coverage_100x is none or (sample.coverage_100x*100)|round(2) == 0 %} 0.0 - {% elif (sample.coverage_100x*100)|round(2) <= {{threshold.coverage_100x_warn }} %} - {% if (sample.coverage_100x*100)|round(2) <= {{threshold.coverage_100x_fail }} %} - {{ (sample.coverage_100x*100)|round(2) }} - {% else %} + {% elif (sample.coverage_100x*100)|round(2) <= threshold.bp_100x_warn %} {{ (sample.coverage_100x*100)|round(2) }} {% else %} {{ (sample.coverage_100x*100)|round(2) }} {% endif %} diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index e22d063d..ba7b97ea 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -270,7 +270,7 @@

Assembly

{% if sample.genome_length > -1 %}

MLST

-
Identitetströskel: {{threshold.mlst_id}}%, Längdtröskel: {{(threshold.mlst_span*100)|round(2)}}%, Novel misstankströskel: {{threshold.mlst_novel_id}}%
+
Identitetströskel: {{threshold.mlst_id}}%, Längdtröskel: {{threshold.mlst_span}}%, Novel misstankströskel: {{threshold.mlst_novel_id}}%
{% if not sample.seq_types|length == 0 and 'Invalid' not in sample.ST_status%} @@ -301,7 +301,7 @@
Identitetströskel: {{threshold.mlst_id}}%, Längdtröskel: {{(threshold.mls

Resistenser

-
Identitetströskel: {{threshold.motif_id}}%, Längdtröskel: {{(threshold.motif_span*100)|round(2)}}%
+
Identitetströskel: {{threshold.motif_id}}%, Längdtröskel: {{threshold.motif_span}}%
{% if not sample.resistances|selectattr('threshold', 'equalto', 'Passed')|list|length == 0 %} From 5f2375e994a18c47ba728c9a38fec3fb60345e40 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 26 Jun 2019 14:02:06 +0200 Subject: [PATCH 33/83] Added Reads threshold. NTC still not accounted for --- .../server/templates/alignment_page.html | 91 ++++++++++--------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index ce71f174..55eb6fb4 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -80,90 +80,93 @@

Provsammanställning

 {% if sample.organism is not none %} - {% else %}{% endif %} + {% else %} + + {% endif %} - - - - - - - - - {% endfor %} From 491e339364fd56e33808b38cc37f878d10cad235 Mon Sep 17 00:00:00 2001 From: sylvinite Date: Wed, 26 Jun 2019 14:05:19 +0200 Subject: [PATCH 34/83] Added default case for non-lims data --- microSALT/server/templates/typing_page.html | 8 ++++++-- microSALT/store/lims_fetcher.py | 14 ++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index e22d063d..48533155 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -73,7 +73,11 @@

Provsammanställning

 - + {% if sample.organism is not none %} + + {% else %} + + {% endif %} {% if sample.date_libprep is not none %} @@ -241,7 +245,7 @@

Översikt

{% endif %} - {% if sample.organism in version %} + {% if sample.organism in version and sample.organism is not none %} diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 4b82e904..79a93031 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -179,8 +179,11 @@ def get_date(self, sample_id, type=""): date_list = date_list + [a.parent_process.date_run for a in arts] except Exception as e: pass - dp = max(date_list).split('-') - return datetime(int(dp[0]), int(dp[1]), int(dp[2])) + if date_list: + dp = max(date_list).split('-') + return datetime(int(dp[0]), int(dp[1]), int(dp[2])) + else: + return datetime.min def get_method(self, sample_id, type=""): """Retrives method document name and version for a sample""" @@ -201,8 +204,11 @@ def get_method(self, sample_id, type=""): arts = self.lims.get_artifacts(samplelimsid = sample_id, process_type = step) processes = [(a.parent_process.udf[key_values['method']], a.parent_process.udf[key_values['version']]) for a in arts] processes = list(set(processes)) - process = sorted(processes)[-1] - out = "{}:{}".format(process[0], process[1]) + if processes: + process = sorted(processes)[-1] + out = "{}:{}".format(process[0], process[1]) except Exception as e: pass + if not 'out' in locals(): + out = "Not in LIMS" return out From 214b446ed7035577bd6b20b662cfd9739fffd7e6 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 26 Jun 2019 15:02:14 +0200 Subject: [PATCH 35/83] Added QC NTC case --- .../server/templates/alignment_page.html | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 55eb6fb4..42c28ef5 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -74,6 +74,7 @@

Provsammanställning

 {% for sample in samples %} + {% if 'Kontroll' in sample.ST_status %}{% endif %} @@ -89,18 +90,28 @@

Provsammanställning

 {% if sample.mapped_rate is none or sample.mapped_rate == 0 %} {% if sample.duplication_rate is none or sample.duplication_rate == -1 or sample.duplication_rate == 0.0 %} {% if sample.insert_size is none or sample.insert_size == 0%} - {% if sample.average_coverage is none or sample.average_coverage|round(2) == 0 %} {% if sample.coverage_10x is none or (sample.coverage_10x*100)|round(2) == 0 %} {% if sample.coverage_30x is none or (sample.coverage_30x*100)|round(2) == 0 %} {% if sample.coverage_50x is none or (sample.coverage_50x*100)|round(2) == 0 %} {% if sample.coverage_100x is none or (sample.coverage_100x*100)|round(2) == 0 %} + {% if 'Kontroll' in sample.ST_status %}{% endif %} {% endfor %}
#{{ sample.application_tag }}{{sample.organism.replace('_', ' ').capitalize()}}UndeterminedUndetermined{{ sample.reference_genome }} + {% if sample.total_reads is none %} - 0 - {% else %} {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{(sample.total_reads/((sample.application_tag[-1:]|float)*10000))|round(2)}}%){% endif %} + 0 + {% else %} + {% set bp_perc = ((sample.total_reads/((sample.application_tag[-1:]|float)*10000))|round(2)) %} + {% if bp_perc < threshold.total_reads_fail %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% elif bp_perc < threshold.total_reads_warn %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% else %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% endif %} + {% endif %} {% if sample.mapped_rate is none or sample.mapped_rate == 0 %} - 0.0 + 0.0 {% elif (sample.mapped_rate*100)|round(2) <= threshold.mapped_rate_warn %} {% if (sample.mapped_rate*100)|round(2) <= threshold.mapped_rate_fail %} - {{ (sample.mapped_rate*100)|round(2) }} + {{ (sample.mapped_rate*100)|round(2) }} {% else %} - {{ (sample.mapped_rate*100)|round(2) }} + {{ (sample.mapped_rate*100)|round(2) }} {% endif %} - {% else %} {{ (sample.mapped_rate*100)|round(2) }} {% endif %} + {% else %} {{ (sample.mapped_rate*100)|round(2) }} {% endif %} - {% if sample.duplication_rate is none or sample.duplication_rate == -1 %} - N/A - {% elif (sample.duplication_rate*100)|round(2) <= threshold.duplication_rate_warn %} - {% if (sample.duplication_rate*100)|round(2) <= threshold.duplication_rate_fail %} - {{ (sample.duplication_rate*100)|round(2) }} + {% if sample.duplication_rate is none or sample.duplication_rate == -1 or sample.duplication_rate == 0.0 %} + N/A + {% elif (sample.duplication_rate*100)|round(2) >= threshold.duplication_rate_warn %} + {% if (sample.duplication_rate*100)|round(2) >= threshold.duplication_rate_fail %} + {{ (sample.duplication_rate*100)|round(2) }} {% else %} - {{ (sample.duplication_rate*100)|round(2) }} + {{ (sample.duplication_rate*100)|round(2) }} {% endif %} {% else %} - {{ (sample.duplication_rate*100)|round(2) }} + {{ (sample.duplication_rate*100)|round(2) }} {% endif %} {% if sample.insert_size is none or sample.insert_size == 0%} - N/A + N/A {% elif sample.insert_size <= threshold.insert_size_warn %} {% if sample.insert_size <= threshold.insert_size_fail %} - {{ insert_size }} + {{ insert_size }} {% else %} - {{ insert_size }} + {{ insert_size }} {% endif %} - {% else %} {{ sample.insert_size }} {% endif %} + {% else %} {{ sample.insert_size }} {% endif %} {% if sample.average_coverage is none or sample.average_coverage|round(2) == 0 %} - 0.0x + 0.0x {% elif (sample.average_coverage)|round(2) <= threshold.average_coverage_warn %} {% if (sample.average_coverage)|round(2) <= threshold.average_coverage_fail %} - {{ (sample.average_coverage)|round(2) }} + {{ (sample.average_coverage)|round(2) }} {% else %} - {{ (sample.average_coverage)|round(2) }} + {{ (sample.average_coverage)|round(2) }} {% endif %} - {% else %} {{ (sample.average_coverage)|round(2) }}x {% endif %} + {% else %} {{ (sample.average_coverage)|round(2) }}x {% endif %} {% if sample.coverage_10x is none or (sample.coverage_10x*100)|round(2) == 0 %} - 0.0 + 0.0 {% elif (sample.coverage_10x*100)|round(2) <= threshold.bp_10x_warn %} {% if (sample.coverage_10x*100)|round(2) <= threshold.bp_10x_fail %} - {{ (sample.coverage_10x*100)|round(2) }} + {{ (sample.coverage_10x*100)|round(2) }} {% else %} - {{ (sample.coverage_10x*100)|round(2) }} + {{ (sample.coverage_10x*100)|round(2) }} {% endif %} - {% else %} {{ (sample.coverage_10x*100)|round(2) }} {% endif %} + {% else %} {{ (sample.coverage_10x*100)|round(2) }} {% endif %} {% if sample.coverage_30x is none or (sample.coverage_30x*100)|round(2) == 0 %} - 0.0 + 0.0 {% elif (sample.coverage_30x*100)|round(2) <= threshold.bp_30x_warn %} - {{ (sample.coverage_30x*100)|round(2) }} - {% else %} {{ (sample.coverage_30x*100)|round(2) }} {% endif %} + {{ (sample.coverage_30x*100)|round(2) }} + {% else %} {{ (sample.coverage_30x*100)|round(2) }} {% endif %} {% if sample.coverage_50x is none or (sample.coverage_50x*100)|round(2) == 0 %} - 0.0 + 0.0 {% elif (sample.coverage_50x*100)|round(2) <= threshold.bp_50x_warn %} - {{ (sample.coverage_50x*100)|round(2) }} - {% else %} {{ (sample.coverage_50x*100)|round(2) }} {% endif %} + {{ (sample.coverage_50x*100)|round(2) }} + {% else %} {{ (sample.coverage_50x*100)|round(2) }} {% endif %} {% if sample.coverage_100x is none or (sample.coverage_100x*100)|round(2) == 0 %} - 0.0 + 0.0 {% elif (sample.coverage_100x*100)|round(2) <= threshold.bp_100x_warn %} - {{ (sample.coverage_100x*100)|round(2) }} - {% else %} {{ (sample.coverage_100x*100)|round(2) }} {% endif %} + {{ (sample.coverage_100x*100)|round(2) }} + {% else %} {{ (sample.coverage_100x*100)|round(2) }} {% endif %}
{{sample.Customer_ID_sample}} {{sample.application_tag}}{{sample.organism.replace('_', ' ').capitalize()}}{{sample.organism.replace('_', ' ').capitalize()}}N/A{{samples[0].projects.date_ordered.date()}}{{sample.date_libprep.date()}} Godkända
Referensversion {{version[sample.organism]}}
{{ sample.Customer_ID_sample }} {{ sample.CG_ID_sample }}0 {% else %} {% set bp_perc = ((sample.total_reads/((sample.application_tag[-1:]|float)*10000))|round(2)) %} - {% if bp_perc < threshold.total_reads_fail %} - {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) - {% elif bp_perc < threshold.total_reads_warn %} - {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% if 'Kontroll' in sample.ST_status %} + {% if bp_perc > threshold.NTC_total_reads_fail %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% elif bp_perc > threshold.NTC_total_reads_warn %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% else %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% endif %} {% else %} - {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) - {% endif %} + {% if bp_perc < threshold.total_reads_fail %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% elif bp_perc < threshold.total_reads_warn %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% else %} + {{'{0:,}'.format(sample.total_reads)|replace(","," ")}} ({{ bp_perc }}%) + {% endif %} + {% endif %} {% endif %} 0.0 - {% elif (sample.mapped_rate*100)|round(2) <= threshold.mapped_rate_warn %} + {% elif (sample.mapped_rate*100)|round(2) <= threshold.mapped_rate_warn and not 'Kontroll' in sample.ST_status %} {% if (sample.mapped_rate*100)|round(2) <= threshold.mapped_rate_fail %} {{ (sample.mapped_rate*100)|round(2) }} {% else %} @@ -110,7 +121,7 @@

Provsammanställning

 N/A - {% elif (sample.duplication_rate*100)|round(2) >= threshold.duplication_rate_warn %} + {% elif (sample.duplication_rate*100)|round(2) >= threshold.duplication_rate_warn and not 'Kontroll' in sample.ST_status %} {% if (sample.duplication_rate*100)|round(2) >= threshold.duplication_rate_fail %} {{ (sample.duplication_rate*100)|round(2) }} {% else %} @@ -121,8 +132,8 @@

Provsammanställning

{% endif %} 		N/A - {% elif sample.insert_size <= threshold.insert_size_warn %} + N/A + {% elif sample.insert_size <= threshold.insert_size_warn and not 'Kontroll' in sample.ST_status %} {% if sample.insert_size <= threshold.insert_size_fail %} {{ insert_size }} {% else %} @@ -132,7 +143,7 @@

Provsammanställning

0.0x - {% elif (sample.average_coverage)|round(2) <= threshold.average_coverage_warn %} + {% elif (sample.average_coverage)|round(2) <= threshold.average_coverage_warn and not 'Kontroll' in sample.ST_status %} {% if (sample.average_coverage)|round(2) <= threshold.average_coverage_fail %} {{ (sample.average_coverage)|round(2) }} {% else %} @@ -142,7 +153,7 @@

Provsammanställning

0.0 - {% elif (sample.coverage_10x*100)|round(2) <= threshold.bp_10x_warn %} + {% elif (sample.coverage_10x*100)|round(2) <= threshold.bp_10x_warn and not 'Kontroll' in sample.ST_status %} {% if (sample.coverage_10x*100)|round(2) <= threshold.bp_10x_fail %} {{ (sample.coverage_10x*100)|round(2) }} {% else %} @@ -152,23 +163,24 @@

Provsammanställning

0.0 - {% elif (sample.coverage_30x*100)|round(2) <= threshold.bp_30x_warn %} + {% elif (sample.coverage_30x*100)|round(2) <= threshold.bp_30x_warn and not 'Kontroll' in sample.ST_status %} {{ (sample.coverage_30x*100)|round(2) }} {% else %} {{ (sample.coverage_30x*100)|round(2) }} {% endif %} 0.0 - {% elif (sample.coverage_50x*100)|round(2) <= threshold.bp_50x_warn %} + {% elif (sample.coverage_50x*100)|round(2) <= threshold.bp_50x_warn and not 'Kontroll' in sample.ST_status %} {{ (sample.coverage_50x*100)|round(2) }} {% else %} {{ (sample.coverage_50x*100)|round(2) }} {% endif %} 0.0 - {% elif (sample.coverage_100x*100)|round(2) <= threshold.bp_100x_warn %} + {% elif (sample.coverage_100x*100)|round(2) <= threshold.bp_100x_warn and not 'Kontroll' in sample.ST_status %} {{ (sample.coverage_100x*100)|round(2) }} {% else %} {{ (sample.coverage_100x*100)|round(2) }} {% endif %}
From 4f0ae8f10b2ea6f4ab7419119c8f81a6cd01c344 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 26 Jun 2019 15:49:04 +0200 Subject: [PATCH 36/83] Added beskrivning for all relevant apptags --- .../server/templates/alignment_page.html | 20 +++++++++++++-- microSALT/server/templates/typing_page.html | 25 +++++++++++++------ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 42c28ef5..51f90811 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -10,7 +10,7 @@ MicroSALT Logo - + Swedac Logo




Kvalitetskontrollsrapport @@ -188,9 +188,25 @@

Provsammanställning

+

Teknisk beskrivning av analysen: {{samples[0].application_tag}}

+

+ {% if samples[0].application_tag == "MWRNXTR003" %} + Mikrobiell helgenomssekvensering av rutinprov, med krav på minst 3 miljoner läspar. + Nextera library preparation. + {% elif samples[0].application_tag in ["MWGNXTR003", "MWMNXTR003", "MWLNXTR003"] %} + Mikrobiell helgenomssekvensering, med krav på minst 3 miljoner läspar. + Nextera library preparation. + {% elif samples[0].application_tag == "MWXNXTR003" %} + Storskalig mikrobiell helgenomssekvensering av minst 176 prover, med krav på minst 3 miljoner läspar. + Nextera library preparation. + {% elif samples[0].application_tag in ["VWGNXTR001", "VWLNXTR001"] %} + Virologisk helgenomssekvensering, med krav på minst 1 miljon läspar. + Nextera library preparation. + {% endif %} +

Analysbegränsningar för: {{samples[0].application_tag}}

- - + Tillförlitligheten hos resultaten förutsätter dels att informationen som bifogats från kund är korrekt.

Avvikelser från metoden

diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index ba7b97ea..2740e7dc 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -10,7 +10,6 @@ MicroSALT Logo - Swedac Logo




Typningsrapport @@ -153,19 +152,29 @@

Resultatsammanställning(***)

- -

Teknisk beskrivning av analysen : {{samples[0].application_tag}}

+ {% if samples[0].application_tag == "MWRNXTR003" %} +

Teknisk beskrivning av analysen: {{samples[0].application_tag}}

+ {% if samples[0].application_tag == "MWRNXTR003" %} Mikrobiell helgenomssekvensering av rutinprov, med krav på minst 3 miljoner läspar. Nextera library preparation. + {% elif samples[0].application_tag in ["MWGNXTR003", "MWMNXTR003", "MWLNXTR003"] %} + Mikrobiell helgenomssekvensering, med krav på minst 3 miljoner läspar. + Nextera library preparation. + {% elif samples[0].application_tag == "MWXNXTR003" %} + Storskalig mikrobiell helgenomssekvensering av minst 176 prover, med krav på minst 3 miljoner läspar. + Nextera library preparation. + {% elif samples[0].application_tag in ["VWGNXTR001", "VWLNXTR001"] %} + Virologisk helgenomssekvensering, med krav på minst 1 miljon läspar. + Nextera library preparation. + {% endif %}

-

Begränsningar av analysen : {{samples[0].application_tag}}

+

Analysbegränsningar för: {{samples[0].application_tag}}

Analysen kan enbart beställas av gruppen Klinisk Mikrobiologi. - Typningen utgår från de publikt tillgängliga databaserna pubMLST och resFinder. - Typningen av prover är enbart att anses som tillförlitlig om desamma uppnår de fördefinierade tröskelvärdena. - Uppnåelse av tröskelvärdena garanteras i sin tur enbart för prover vars förhantering och detaljer tidigare verifierats av personal på Clinical Genomics. + Typningen begränsas av den information som vid analys återfinns i de publikt tillgängliga databaserna pubMLST och resFinder. + Tillförlitligheten hos resultaten förutsätter dels att informationen som bifogats från kund är korrekt. Dels att proverna uppnår de fördefinierade tröskelvärdena; och dels att de organismerna som analyseras har tidigare manuellt verifierats tidigare av personal på Clinical Genomics.

Avvikelser från metoden

@@ -181,7 +190,7 @@

Signatur för godkännande av rapport

Valtteri Wirta
Head of unit, Clinical Genomics

- + {% endif %}
From 24f43391380e5d97f25934bf516d770aa98250e4 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 27 Jun 2019 15:10:31 +0200 Subject: [PATCH 37/83] Added unused steps and reports tabls --- microSALT/store/db_manipulator.py | 11 +++++++---- microSALT/store/orm_models.py | 26 +++++++++++++++++++------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index af206a6b..b6415506 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -11,7 +11,7 @@ # maintain the same connection per thread from sqlalchemy.pool import SingletonThreadPool -from microSALT.store.orm_models import app, Projects, Resistances, Samples, Seq_types, Versions +from microSALT.store.orm_models import app, Projects, Reports, Resistances, Samples, Seq_types, Steps, Versions from microSALT.store.models import Profiles, Novel class DB_Manipulator: @@ -47,9 +47,12 @@ def create_tables(self): if not self.engine.dialect.has_table(self.engine, 'resistances'): Resistances.__table__.create(self.engine) self.logger.info("Created resistance table") -# if not self.engine.dialect.has_table(self.engine, 'steps'): -# Steps.__table__.create(self.engine) -# self.logger.info("Created step table") + if not self.engine.dialect.has_table(self.engine, 'steps'): + Steps.__table__.create(self.engine) + self.logger.info("Created step table") + if not self.engine.dialect.has_table(self.engine, 'reports'): + Reports.__table__.create(self.engine) + self.logger.info("Created reports table") for k,v in self.profiles.items(): if not self.engine.dialect.has_table(self.engine, "profile_{}".format(k)): self.profiles[k].create() diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index 3c15e874..03eb5c66 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -87,13 +87,15 @@ class Resistances(db.Model): contig_start=db.Column(db.Integer) contig_end=db.Column(db.Integer) -#Debug: Multi-date support for libprep/sequencing -#class Steps(db.Model): -# __tablename__ = 'steps' -# CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) -# date = db.Column(db.DateTime) -# method = db.Column(db.String(40)) -# step = db.Column(db.String(40)) +#Multi-date support for libprep/sequencing/analysis +class Steps(db.Model): + __tablename__ = 'steps' + samples= relationship("Samples", back_populates="steps") + + CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) + step = db.Column(db.String(40)) + method = db.Column(db.String(40)) + date = db.Column(db.DateTime) class Projects(db.Model): __tablename__ = 'projects' @@ -109,3 +111,13 @@ class Versions(db.Model): name = db.Column(db.String(45), primary_key=True, nullable=False) version = db.Column(db.String(10)) + +#Keeps and aggregate step string, makes a new version whenever one is not found +class Reports(db.Model): + __tablename__ = 'reports' + projects = relationship('Projects', back_populates='reports') + + CG_ID_project = db.Column(db.String(15), ForeignKey('projects.CG_ID_project')) + steps_aggregate = db.Column(db.String(100)) + date = db.Column(db.DateTime) + version = db.Column(db.Integer, default=1) From 5b8e18176b13b6aa6c8408ff599deb42e0ddc63c Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 27 Jun 2019 15:20:21 +0200 Subject: [PATCH 38/83] Reports now copied to general reports folder --- configExample.json | 2 ++ microSALT/utils/reporter.py | 29 +++++++++++++++++++++-------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/configExample.json b/configExample.json index 219d5d66..b72daf04 100644 --- a/configExample.json +++ b/configExample.json @@ -19,6 +19,8 @@ "folders": { "_comment": "Root folder for ALL output", "results": "/mnt/hds/proj/bioinfo/MICROBIAL/MLST/results/", + "_comment": "Report collection folder", + "reports": "/mnt/hds/proj/bioinfo/MICROBIAL/MLST/reports/", "_comment": "Log file position and name", "log_file": "~/microsalt.log", diff --git a/microSALT/utils/reporter.py b/microSALT/utils/reporter.py index 4a4f0e45..02077cf3 100644 --- a/microSALT/utils/reporter.py +++ b/microSALT/utils/reporter.py @@ -11,6 +11,7 @@ import smtplib from datetime import datetime +from shutil import copyfile from os.path import basename from email.mime.text import MIMEText @@ -92,11 +93,17 @@ def gen_qc(self,silent=False): sys.exit(-1) try: q = requests.get("http://127.0.0.1:5000/microSALT/{}/qc".format(self.name), allow_redirects=True) - outqc = "{}/{}_QC_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) - open(outqc, 'wb').write(q.content.decode("iso-8859-1").encode("utf8")) - self.filelist.append(outqc) + outfile = "{}_QC_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) + output ="{}/{}".format(self.output, outfile) + storage = "{}/{}".format(self.config['folders']['reports'], outfile) + + open(output, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) + copyfile(primary, storage) + + self.filelist.append(output) + self.filelist.append(storage) if not silent: - self.attachments.append(outqc) + self.attachments.append(output) except Exception as e: self.logger.error("Flask instance currently occupied. Possible rogue process. Retry command") self.error = True @@ -110,11 +117,17 @@ def gen_typing(self,silent=False): sys.exit(-1) try: r = requests.get("http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), allow_redirects=True) - outtype = "{}/{}_Typing_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) - open(outtype, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) - self.filelist.append(outtype) + outfile = "{}/{}_Typing_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) + output ="{}/{}".format(self.output, outfile) + storage = "{}/{}".format(self.config['folders']['reports'], outfile) + + open(output, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) + copyfile(primary, storage) + + self.filelist.append(output) + self.filelist.append(storage) if not silent: - self.attachments.append(outtype) + self.attachments.append(output) except Exception as e: self.logger.error("Flask instance currently occupied. Possible rogue process. Retry command") self.error = True From bcc5e3c9a668ee16f51494adab6b1e5e130640bb Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 27 Jun 2019 16:08:02 +0200 Subject: [PATCH 39/83] Added fastq size warning --- microSALT/utils/job_creator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index be49b756..64e81f42 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -92,6 +92,12 @@ def verify_fastq(self): raise Exception("Some fastq files have no mate in directory {}.".format(self.indir)) if verified_files == []: raise Exception("No files in directory {} match file_pattern '{}'.".format(self.indir, self.config['regex']['file_pattern'])) + #Warn about file sizes + for vfile in verified_files: + bsize = os.stat("{}/{}".format(self.indir,vfile).st_size + bsize = bsize >> 20 + if bsize > 1000: + self.logger.warning("Input fastq {} exceeds 1000MB") return verified_files def create_assemblysection(self): From c44209fad30c4b26fd66517e82fe51f08fe1ec48 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 10:27:30 +0200 Subject: [PATCH 40/83] Invalid ST/thres/organism now in swedish. qc_only wont generate typing report --- microSALT/server/templates/alignment_page.html | 2 +- microSALT/server/templates/typing_page.html | 6 +++++- microSALT/server/views.py | 4 ++-- microSALT/utils/job_creator.py | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 7e0ca967..ce8d4d36 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -80,7 +80,7 @@

Provsammanställning

{{ sample.application_tag }} {% if sample.organism is not none %} {{sample.organism.replace('_', ' ').capitalize()}} - {% else %}Undetermined{% endif %} + {% else %}Okänd{% endif %} {{ sample.reference_genome }} {% if sample.total_reads is none %} diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 48533155..7571909a 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -145,6 +145,8 @@

Resultatsammanställning(***)

{% endif %} {% if sample.threshold == 'Failed' %} Underkända + {% elif sample.threshold == '-' %} + - {% else %} Godkända {% endif %} @@ -241,6 +243,8 @@

Översikt

Tröskelvärden {% if sample.threshold=='Failed' %} Underkända + {% elif sample.threshold == '-' %} + - {% else %} Godkända {% endif %} @@ -276,7 +280,7 @@

Assembly

MLST

Identitetströskel: {{threshold.mlst_id}}%, Längdtröskel: {{(threshold.mlst_span*100)|round(2)}}%, Novel misstankströskel: {{threshold.mlst_novel_id}}%
- {% if not sample.seq_types|length == 0 and 'Invalid' not in sample.ST_status%} + {% if not sample.seq_types|length == 0 and 'saknas' not in sample.ST_status%} diff --git a/microSALT/server/views.py b/microSALT/server/views.py index f7a1369f..0d6da614 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -114,7 +114,7 @@ def gen_reportdata(pid='all', organism_group='all'): s.Customer_ID_sample.startswith('blank') or s.Customer_ID_sample.startswith('dual-NTC'): s.ST_status = 'Kontroll (prefix)' - if 'Kontroll' in s.ST_status or 'Control' in s.ST_status or s.ST == -1: + if ('Kontroll' in s.ST_status or 'Control' in s.ST_status) or s.ST == -1: s.threshold = '-' elif s.ST == -3: s.threshold = 'Failed' @@ -137,7 +137,7 @@ def gen_reportdata(pid='all', organism_group='all'): if not ('Control' in s.ST_status or 'Kontroll' in s.ST_status) and s.ST < 0: if s.ST == -1: - s.ST_status = 'Invalid data' + s.ST_status = 'Data saknas' elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Passed': s.ST_status = 'Novel' elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Failed': diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index be49b756..b693a735 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -405,8 +405,8 @@ def finish_job(self, joblist, single_sample=False): if 'config_path' in self.config: custom_conf = '--config {}'.format(self.config['config_path']) - mb.write("microSALT utils finish {} {} --input {} --rerun --email {} {}\n".\ - format(span, self.name, self.finishdir, self.config['regex']['mail_recipient'], custom_conf)) + mb.write("microSALT utils finish {} {} --input {} --rerun --email {} --report {} {}\n".\ + format(span, self.name, self.finishdir, self.config['regex']['mail_recipient'], report, custom_conf)) mb.write("touch {}/run_complete.out".format(self.finishdir)) mb.close() From b282ed6b2d1d84f78327a4e0c53c0c1c18c622bd Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 10:35:37 +0200 Subject: [PATCH 41/83] Swedified not in lims --- microSALT/server/templates/typing_page.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 7571909a..1ea06c6b 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -79,16 +79,16 @@

Provsammanställning

 {% endif %} - {% if sample.date_libprep is not none %} - - {% else %} + {% if sample.date_libprep is not none or 'Not in LIMS' in sample.date_libprep %} + {% else %} + {% endif %} - {% if sample.date_sequencing is not none %} - - {% else %} + {% if sample.date_sequencing is not none 'Not in LIMS' in sample.date_sequencing %} + {% else %} + {% endif %} From 11d2fdfdf317fb61b9be19dea80631d7802226f3 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 11:15:57 +0200 Subject: [PATCH 42/83] Added reports pk --- microSALT/store/orm_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index 03eb5c66..4686e9be 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -117,7 +117,7 @@ class Reports(db.Model): __tablename__ = 'reports' projects = relationship('Projects', back_populates='reports') - CG_ID_project = db.Column(db.String(15), ForeignKey('projects.CG_ID_project')) + CG_ID_project = db.Column(db.String(15), ForeignKey('projects.CG_ID_project'), primary_key=True) steps_aggregate = db.Column(db.String(100)) date = db.Column(db.DateTime) version = db.Column(db.Integer, default=1) From 6275ab718835ded14f779b23ecaffba67de745c1 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 11:28:22 +0200 Subject: [PATCH 43/83] Syntax error fix --- microSALT/utils/job_creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 64e81f42..cfdf19f0 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -94,7 +94,7 @@ def verify_fastq(self): raise Exception("No files in directory {} match file_pattern '{}'.".format(self.indir, self.config['regex']['file_pattern'])) #Warn about file sizes for vfile in verified_files: - bsize = os.stat("{}/{}".format(self.indir,vfile).st_size + bsize = os.stat("{}/{}".format(self.indir,vfile)).st_size bsize = bsize >> 20 if bsize > 1000: self.logger.warning("Input fastq {} exceeds 1000MB") From 69e9aa2ce4d35edfbd4e9e1556a35baa3e9dc48e Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 12:53:26 +0200 Subject: [PATCH 44/83] Syntax error fix --- microSALT/server/templates/typing_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 1ea06c6b..10574504 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -85,7 +85,7 @@

Provsammanställning

 {% endif %} - {% if sample.date_sequencing is not none 'Not in LIMS' in sample.date_sequencing %} + {% if sample.date_sequencing is not none or 'Not in LIMS' in sample.date_sequencing %} {% else %} From 1de6778eb4d23c8be913badeb9d58bfa6e0c7542 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 13:02:19 +0200 Subject: [PATCH 45/83] Internal sample check now falls back to external names --- microSALT/store/lims_fetcher.py | 45 +++++++++++++++++---------------- microSALT/utils/job_creator.py | 2 +- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 4b82e904..51eaa94e 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -42,39 +42,40 @@ def load_lims_project_info(self, cg_projid): except KeyError as e: self.logger.warn("Unable to fetch LIMS info for project {}\nSource: {}".format(cg_projid, str(e))) - def samples_in_project(self, cg_projid, external=False): + def samples_in_project(self, cg_projid): """ Returns a list of sample names for a project""" output = list() - if not external: - samples = self.lims.get_samples(projectlimsid=cg_projid) - else: - samples = self.lims.get_samples(projectname=cg_projid) + #Uses internal names, then external on empty + samples = self.lims.get_samples(projectlimsid=cg_projid) + if not samples: + samples = self.lims.get_samples(projectname=cg_projid) + for s in samples: output.append(s.id) return output - def load_lims_sample_info(self, cg_sampleid, external=False): + def load_lims_sample_info(self, cg_sampleid): """ Loads all utilized LIMS info. Organism assumed to be written as binomial name """ libprep_date = "" seq_date = "" - if external: - sample = self.lims.get_samples(name=cg_sampleid) - if len(sample) != 1: - self.logger.error("Sample ID {} resolves to multiple entries".format(cg_sampleid)) - sample = sample[0] + + try: + #Internal + sample = Sample(self.lims, id=cg_sampleid) + #External + if not sample: + sample = self.lims.get_samples(name=cg_sampleid) + if len(sample) != 1: + self.logger.error("Sample ID {} resolves to multiple entries".format(cg_sampleid + sample = sample[0] + method_libprep = self.get_method(cg_sampleid,type='libprep') method_sequencing = self.get_method(cg_sampleid,type='sequencing') date_libprep = self.get_date(cg_sampleid,type="libprep") date_sequencing = self.get_date(cg_sampleid,type="sequencing") - else: - try: - sample = Sample(self.lims, id=cg_sampleid) - method_libprep = self.get_method(cg_sampleid,type='libprep') - method_sequencing = self.get_method(cg_sampleid,type='sequencing') - date_libprep = self.get_date(cg_sampleid,type="libprep") - date_sequencing = self.get_date(cg_sampleid,type="sequencing") - except Exception as e: - self.logger.error("LIMS connection timeout") + except Exception as e: + self.logger.error("LIMS connection timeout") + organism = "Unset" if 'Strain' in sample.udf and organism == "Unset": #Predefined genus usage. All hail buggy excel files @@ -139,10 +140,10 @@ def load_lims_sample_info(self, cg_sampleid, external=False): self.logger.warn("Unable to fetch LIMS info for sample {}. Review LIMS data.\nSource: {}"\ .format(cg_sampleid, str(e))) - def get_organism_refname(self, sample_name, external=False): + def get_organism_refname(self, sample_name): """Finds which reference contains the same words as the LIMS reference and returns it in a format for database calls.""" - self.load_lims_sample_info(sample_name, external) + self.load_lims_sample_info(sample_name) lims_organ = self.data['organism'].lower() orgs = os.listdir(self.config["folders"]["references"]) organism = re.split('\W+', lims_organ) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index cfdf19f0..8d735279 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -456,7 +456,7 @@ def sample_job(self): if not os.path.exists(self.finishdir): os.makedirs(self.finishdir) try: - self.organism = self.lims_fetcher.get_organism_refname(self.name, external=False) + self.organism = self.lims_fetcher.get_organism_refname(self.name) # This is one job self.batchfile = "{}/runfile.sbatch".format(self.finishdir) batchfile = open(self.batchfile, "w+") From 703d67cb2ed750e5229360006f7c4c3b4dc17419 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 13:10:02 +0200 Subject: [PATCH 46/83] Reversed if case, added safety to methods --- microSALT/server/templates/typing_page.html | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 10574504..975c7fa6 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -79,18 +79,26 @@

Provsammanställning

 {% endif %} - {% if sample.date_libprep is not none or 'Not in LIMS' in sample.date_libprep %} - + {% if sample.date_libprep is none or 'Not in LIMS' in sample.date_libprep %} + {% else %} {% endif %} - - {% if sample.date_sequencing is not none or 'Not in LIMS' in sample.date_sequencing %} - + {% if sample.method_libprep is none or 'Not in LIMS' in sample.method_libprep %} + + {% else %} + + {% endif %} + {% if sample.date_sequencing is none or 'Not in LIMS' in sample.date_sequencing %} + {% else %} {% endif %} - + {% if sample.method_sequencing is none or 'Not in LIMS' in sample.method_sequencing %} + + {% else %} + + {% endif %} {% if sample.priority == 'standard' %} From a6a2144f56afccb74c2c8dec565efaac3a2f10eb Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 13:19:45 +0200 Subject: [PATCH 47/83] Fixed iffy none logic --- microSALT/server/templates/typing_page.html | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 975c7fa6..ce1f49b7 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -79,25 +79,25 @@

Provsammanställning

 {% endif %} - {% if sample.date_libprep is none or 'Not in LIMS' in sample.date_libprep %} - - {% else %} + {% if sample.date_libprep is not none and not 'Not in LIMS' in sample.date_libprep %} - {% endif %} - {% if sample.method_libprep is none or 'Not in LIMS' in sample.method_libprep %} - {% else %} - - {% endif %} - {% if sample.date_sequencing is none or 'Not in LIMS' in sample.date_sequencing %} - {% else %} - {% endif %} - {% if sample.method_sequencing is none or 'Not in LIMS' in sample.method_sequencing %} + {% if sample.method_libprep is not none and not 'Not in LIMS' in sample.method_libprep %} + + {% else %} + {% endif %} + {% if sample.date_sequencing is not none and not 'Not in LIMS' in sample.date_sequencing %} + {% else %} + + {% endif %} + {% if sample.method_sequencing is not none and not 'Not in LIMS' in sample.method_sequencing %} + {% else %} + {% endif %} {% if sample.priority == 'standard' %} From 8144f848dbfd780aa74c5d9e1fe8a8d3f6566721 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 1 Jul 2019 13:50:55 +0200 Subject: [PATCH 48/83] Typing report finally working as intended again --- microSALT/server/templates/typing_page.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index ce1f49b7..3c3e5ca4 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -79,22 +79,22 @@

Provsammanställning

 {% endif %} - {% if sample.date_libprep is not none and not 'Not in LIMS' in sample.date_libprep %} + {% if sample.date_libprep is not none %} {% else %} {% endif %} - {% if sample.method_libprep is not none and not 'Not in LIMS' in sample.method_libprep %} + {% if sample.method_libprep is not none and 'Not in LIMS' not in sample.method_libprep %} {% else %} {% endif %} - {% if sample.date_sequencing is not none and not 'Not in LIMS' in sample.date_sequencing %} + {% if sample.date_sequencing is not none %} {% else %} {% endif %} - {% if sample.method_sequencing is not none and not 'Not in LIMS' in sample.method_sequencing %} + {% if sample.method_sequencing is not none and 'Not in LIMS' not in sample.method_sequencing %} {% else %} From a8c059553762201c26d66464df49365d0163b26c Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Tue, 2 Jul 2019 09:49:13 +0200 Subject: [PATCH 49/83] Analysis collection start, attempt 1 --- microSALT/cli.py | 63 +++++++++++++++++++++++++++++++++ microSALT/store/lims_fetcher.py | 2 +- microSALT/utils/job_creator.py | 35 ++++++++++++++++-- 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/microSALT/cli.py b/microSALT/cli.py index fc7f77ad..3db17643 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -187,6 +187,69 @@ def sample(ctx, sample_id, input, dry, config, email, qc_only, untrimmed, skip_u click.echo("Unable to process sample {} due to '{}'".format(sample_id,e)) done() + +#WIP WIP +@analyse.command() +@click.argument('collection_id') +@click.option('--input', help='Full path to sample folder', default="") +@click.option('--dry', help="Builds instance without posting to SLURM", default=False, is_flag=True) +@click.option('--qc_only', help="Only runs QC (alignment stats)", default=False, is_flag=True) +@click.option('--config', help="microSALT config to override default", default="") +@click.option('--email', default=config['regex']['mail_recipient'], help='Forced e-mail recipient') +@click.option('--untrimmed', help="Use untrimmed input data", default=False, is_flag=True) +@click.option('--skip_update', default=False, help="Skips downloading of references", is_flag=True) +@click.option('--uncareful', help="Avoids running SPAdes in careful mode. Sometimes fix assemblies", default=False, is_flag=True) +@click.pass_context +def collection(ctx, collection_id, input, dry, config, email, qc_only, untrimmed, skip_update, uncareful): + """Analyse a collection of samples""" + ctx.obj['config']['regex']['mail_recipient'] = email + trimmed = not untrimmed + careful = not uncareful + if config != '': + if os.path.exists(config): + try: + with open(os.path.abspath(config), 'r') as conf: + ctx.obj['config'] = json.load(conf) + ctx.obj['config']['config_path'] = os.path.abspath(config) + except Exception as e: + pass + + ctx.obj['config']['dry'] = dry + scientist=LIMS_Fetcher(ctx.obj['config'], ctx.obj['log']) + + pool = list() + if input != "": + collection_dir = os.path.abspath(input) + for sample in os.listdir(input): + if os.path.isdir(sample): + pool.append(sample) + try: + scientist.load_lims_sample_info(sample) + except Exception as e: + click.echo("Unable to load LIMS sample info for sample {}.".format(sample)) + sys.exit(-1) + + click.echo("Checking versions of references..") + for sample in pool: + try: + if not skip_update: + fixer = Referencer(ctx.obj['config'], ctx.obj['log']) + fixer.identify_new(sample,project=False) + fixer.update_refs() + print("Version check done. Creating sbatch job") + else: + print("Skipping version check.") + except Exception as e: + click.echo("Unable to update references for sample {} due to '{}'".format(sample,e)) + + #APPLY POOL IN A NEAT WAY + try: + worker = Job_Creator(sample_dir, ctx.obj['config'], ctx.obj['log'], trim=trimmed,qc_only=qc_only, careful=careful, pool=pool) + worker.project_job(pool=True) + except Exception as e: + click.echo("Unable to process sample {} due to '{}'".format(sample_id,e)) + done() + @finish.command() @click.argument('sample_id') @click.option('--rerun', is_flag=True, default=False, help='Overwrite existing data') diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 51eaa94e..4b32f970 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -66,7 +66,7 @@ def load_lims_sample_info(self, cg_sampleid): if not sample: sample = self.lims.get_samples(name=cg_sampleid) if len(sample) != 1: - self.logger.error("Sample ID {} resolves to multiple entries".format(cg_sampleid + self.logger.error("Sample ID {} resolves to multiple entries".format(cg_sampleid)) sample = sample[0] method_libprep = self.get_method(cg_sampleid,type='libprep') diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 8d735279..db248375 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -16,7 +16,7 @@ class Job_Creator(): - def __init__(self, input, config, log, finishdir="", timestamp="", trim=True, qc_only=False,careful=False): + def __init__(self, input, config, log, finishdir="", timestamp="", trim=True, qc_only=False,careful=False,pool=list()): self.config = config self.logger = log self.batchfile = "" @@ -25,6 +25,7 @@ def __init__(self, input, config, log, finishdir="", timestamp="", trim=True, qc self.trimmed=trim self.qc_only = qc_only self.careful = careful + self.pool = pool if isinstance(input, str): self.indir = os.path.abspath(input) @@ -334,14 +335,22 @@ def project_job(self, single_sample=False): jobarray = list() if not os.path.exists(self.finishdir): os.makedirs(self.finishdir) + #Loads project level info. try: if single_sample: self.create_project(os.path.normpath(self.indir).split('/')[-2]) + elif self.pool: + addedprojs = list() + for sample in pool: + proj = sample[:-3] + if proj not in addedprojs: + self.create_project(proj) + addedprojs.append(proj) else: self.create_project(self.name) except Exception as e: self.logger.error("LIMS interaction failed. Unable to read/write project {}".format(self.name)) - #Start every sample job + #Writes the job creation sbatch if single_sample: try: self.sample_job() @@ -357,6 +366,28 @@ def project_job(self, single_sample=False): self.logger.info("Suppressed command: {}".format(bash_cmd)) except Exception as e: self.logger.error("Unable to analyze single sample {}".format(self.name)) + elif self.pool: + for (dirpath, dirnames, filenames) in os.walk(self.indir): + for dir in dirnames: + try: + sample_in = "{}/{}".format(dirpath, dir) + sample_out = "{}/{}".format(self.finishdir, dir) + sample_instance = Job_Creator(sample_in, self.config, self.logger, sample_out, self.now, trim=self.trimmed) + sample_instance.sample_job() + headerargs = sample_instance.get_headerargs() + outfile = "" + if os.path.isfile(sample_instance.get_sbatch()): + outfile = sample_instance.get_sbatch() + bash_cmd="sbatch {} {}".format(headerargs, outfile) + if not dry and outfile != "": + projproc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) + output, error = projproc.communicate() + jobno = re.search('(\d+)', str(output)).group(0) + jobarray.append(jobno) + else: + self.logger.info("Suppressed command: {}".format(bash_cmd)) + except Exception as e: + pass else: for (dirpath, dirnames, filenames) in os.walk(self.indir): for dir in dirnames: From d2d80ab16b3930d54304e9c686ac40c780d44690 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 11:01:17 +0200 Subject: [PATCH 50/83] Analysis collection start, attempt 2 --- microSALT/cli.py | 114 +++++++++++++++++++++----- microSALT/server/views.py | 132 +++++++++++++++++------------- microSALT/store/db_manipulator.py | 12 ++- microSALT/store/lims_fetcher.py | 40 +++++---- microSALT/utils/job_creator.py | 12 ++- microSALT/utils/reporter.py | 34 +++++--- microSALT/utils/scraper.py | 4 +- 7 files changed, 229 insertions(+), 119 deletions(-) diff --git a/microSALT/cli.py b/microSALT/cli.py index 3db17643..4eef6f97 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -188,7 +188,6 @@ def sample(ctx, sample_id, input, dry, config, email, qc_only, untrimmed, skip_u done() -#WIP WIP @analyse.command() @click.argument('collection_id') @click.option('--input', help='Full path to sample folder', default="") @@ -200,7 +199,7 @@ def sample(ctx, sample_id, input, dry, config, email, qc_only, untrimmed, skip_u @click.option('--skip_update', default=False, help="Skips downloading of references", is_flag=True) @click.option('--uncareful', help="Avoids running SPAdes in careful mode. Sometimes fix assemblies", default=False, is_flag=True) @click.pass_context -def collection(ctx, collection_id, input, dry, config, email, qc_only, untrimmed, skip_update, uncareful): +def collection(ctx, collection_id, input, dry, qc_only, config, email, untrimmed, skip_update, uncareful): """Analyse a collection of samples""" ctx.obj['config']['regex']['mail_recipient'] = email trimmed = not untrimmed @@ -217,37 +216,46 @@ def collection(ctx, collection_id, input, dry, config, email, qc_only, untrimmed ctx.obj['config']['dry'] = dry scientist=LIMS_Fetcher(ctx.obj['config'], ctx.obj['log']) - pool = list() + pool = [] if input != "": collection_dir = os.path.abspath(input) - for sample in os.listdir(input): - if os.path.isdir(sample): - pool.append(sample) - try: - scientist.load_lims_sample_info(sample) - except Exception as e: - click.echo("Unable to load LIMS sample info for sample {}.".format(sample)) + else: + collection_dir = "{}/{}".format(ctx.obj['config']['folders']['seqdata'], collection_id) + for sample in os.listdir(collection_dir): + if os.path.isdir("{}/{}".format(collection_dir,sample)): + pool.append(sample) + if pool == []: + click.echo("Input collection lacks any valid of samples") sys.exit(-1) - click.echo("Checking versions of references..") + pool_cg = [] for sample in pool: + try: + scientist.load_lims_sample_info(sample,warnings=True) + pool_cg.append(scientist.data['CG_ID_sample']) + except Exception as e: + click.echo("Unable to load LIMS sample info for sample {}.".format(sample)) + sys.exit(-1) + + click.echo("Checking versions of references..") + for sample in pool_cg: try: if not skip_update: fixer = Referencer(ctx.obj['config'], ctx.obj['log']) fixer.identify_new(sample,project=False) - fixer.update_refs() - print("Version check done. Creating sbatch job") - else: - print("Skipping version check.") except Exception as e: - click.echo("Unable to update references for sample {} due to '{}'".format(sample,e)) + click.echo("Unable to update references for sample {} due to '{}'".format(sample,str(e))) + if not skip_update: + fixer.update_refs() + print("Version check done. Creating sbatch job") + else: + print("Skipping version check.") - #APPLY POOL IN A NEAT WAY try: - worker = Job_Creator(sample_dir, ctx.obj['config'], ctx.obj['log'], trim=trimmed,qc_only=qc_only, careful=careful, pool=pool) - worker.project_job(pool=True) + worker = Job_Creator(collection_dir, ctx.obj['config'], ctx.obj['log'], trim=trimmed,qc_only=qc_only, careful=careful, pool=pool_cg) + worker.project_job() except Exception as e: - click.echo("Unable to process sample {} due to '{}'".format(sample_id,e)) + click.echo("Unable to process collection due to '{}'".format(str(e))) done() @finish.command() @@ -346,6 +354,67 @@ def project(ctx, project_id, rerun, email, input, config, report): codemonkey.report(report) done() +@finish.command() +@click.argument('collection_id') +@click.option('--rerun', is_flag=True, default=False, help='Overwrite existing data') +@click.option('--email', default=config['regex']['mail_recipient'], help='Forced e-mail recipient') +@click.option('--input', help='Full path to result sample folder', default="") +@click.option('--config', help="microSALT config to override default", default="") +@click.option('--report', default='default', type=click.Choice(['default', 'qc'])) +@click.pass_context +def collection(ctx, collection_id, rerun, email, input, config, report): + """Parse results from analysing a set of sample""" + if config != '': + if os.path.exists(config): + try: + with open(os.path.abspath(config), 'r') as conf: + ctx.obj['config'] = json.load(conf) + ctx.obj['config']['config_path'] = os.path.abspath(config) + except Exception as e: + pass + + ctx.obj['config']['rerun'] = rerun + ctx.obj['config']['regex']['mail_recipient'] = email + + pool = [] + if input != "": + collection_dir = os.path.abspath(input) + else: + prohits = [x for x in os.listdir(ctx.obj['config']['folders']['results']) if x.startswith("{}_".format(collection_id))] + if len(prohits) > 1: + click.echo("Multiple instances of that analysis exists. Specify full path using --input") + sys.exit(-1) + elif len(prohits) <1: + click.echo("No analysis folder prefixed by {} found.".format(project_id)) + sys.exit(-1) + else: + collection_dir = "{}/{}".format(ctx.obj['config']['folders']['results'], prohits[-1]) + + for sample in os.listdir(collection_dir): + if os.path.isdir("{}/{}".format(collection_dir,sample)): + pool.append(sample) + if pool == []: + click.echo("Input collection lacks any valid of samples") + sys.exit(-1) + + pool_cg = [] + scientist=LIMS_Fetcher(ctx.obj['config'], ctx.obj['log']) + for sample in pool: + try: + scientist.load_lims_sample_info(sample,warnings=True) + pool_cg.append(scientist.data['CG_ID_sample']) + except Exception as e: + click.echo("Unable to load LIMS sample info for sample {}.".format(sample)) + sys.exit(-1) + + for sample in pool: + garbageman = Scraper("{}/{}".format(collection_dir, sample), ctx.obj['config'], ctx.obj['log']) + garbageman.scrape_sample() + codemonkey = Reporter(ctx.obj['config'], ctx.obj['log'], collection_id, output=collection_dir, collection=True) + codemonkey.report(report) + done() + + @refer.command() @click.argument('organism') @click.pass_context @@ -376,11 +445,12 @@ def list(ctx): @click.option('--email', default=config['regex']['mail_recipient'], help='Forced e-mail recipient') @click.option('--type', default='default', type=click.Choice(['default', 'typing', 'resistance_overview', 'qc', 'json_dump', 'st_update'])) @click.option('--output',help='Full path to output folder',default="") +@click.option('--collection',default=False, is_flag=True) @click.pass_context -def report(ctx, project_name, email, type, output): +def report(ctx, project_name, email, type, output, collection): """Re-generates report for a project""" ctx.obj['config']['regex']['mail_recipient'] = email - codemonkey = Reporter(ctx.obj['config'], ctx.obj['log'], project_name, output) + codemonkey = Reporter(ctx.obj['config'], ctx.obj['log'], project_name, output, collection=collection) codemonkey.report(type) done() diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 9b6a04e3..85d604df 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -10,7 +10,7 @@ from microSALT import config, __version__ from microSALT.store.db_manipulator import app -from microSALT.store.orm_models import Projects, Samples, Seq_types, Versions +from microSALT.store.orm_models import Collections, Projects, Samples, Seq_types, Versions engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], connect_args={'check_same_thread': False}) Session = sessionmaker(bind=engine) @@ -86,6 +86,16 @@ def STtracker_page(customer): return render_template('STtracker_page.html', internal = final_samples) +def gen_collectiondata(samples=[]): + """ Queries database using a set of samples""" + session.query(Collections).filter(Collections.ID_collection==samples) + print("TO BE IMPLEMENTED") + + #prefix_sam = ["CG_ID_sample=="+ x for x in samples] + #sample_info = session.query(Samples).filter(or_(','.join(prefix_sam))) + + #sample_info = gen_add_info(sample_info) + def gen_reportdata(pid='all', organism_group='all'): """ Queries database for all necessary information for the reports """ output = dict() @@ -105,62 +115,66 @@ def gen_reportdata(pid='all', organism_group='all'): sample_info = sorted(sample_info, key=lambda sample: \ int(sample.CG_ID_sample.replace(sample.CG_ID_project, '')[1:])) - #Set ST status - for s in sample_info: - s.ST_status=str(s.ST) - if s.Customer_ID_sample.startswith('NTC') or s.Customer_ID_sample.startswith('0-') or \ - s.Customer_ID_sample.startswith('NK-') or s.Customer_ID_sample.startswith('NEG') or \ - s.Customer_ID_sample.startswith('CTRL') or s.Customer_ID_sample.startswith('Neg') or \ - s.Customer_ID_sample.startswith('blank') or s.Customer_ID_sample.startswith('dual-NTC'): - s.ST_status = 'Kontroll (prefix)' - - if 'Kontroll' in s.ST_status or 'Control' in s.ST_status or s.ST == -1: - s.threshold = '-' - elif s.ST == -3: - s.threshold = 'Failed' - elif hasattr(s, 'seq_types') and s.seq_types != [] or s.ST == -2: - near_hits=0 - s.threshold = 'Passed' - for seq_type in s.seq_types: - #Identify single deviating allele - if seq_type.st_predictor and seq_type.identity >= config["threshold"]["mlst_novel_id"] and \ - config["threshold"]["mlst_id"] > seq_type.identity and 1-abs(1-seq_type.span) >= (config["threshold"]["mlst_span"]/100.0): - near_hits = near_hits + 1 - elif (seq_type.identity < config["threshold"]["mlst_novel_id"] or \ - seq_type.span < (config["threshold"]["mlst_span"]/100.0)) and seq_type.st_predictor: - s.threshold = 'Failed' - - if near_hits > 0 and s.threshold == 'Passed': - s.ST_status = 'Unknown ({} alleles)'.format(near_hits) - else: - s.threshold = 'Failed' - - if not ('Control' in s.ST_status or 'Kontroll' in s.ST_status) and s.ST < 0: - if s.ST == -1: - s.ST_status = 'Invalid data' - elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Passed': - s.ST_status = 'Novel' - elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Failed': - s.ST_status = 'Unknown' - else: - s.ST_status='None' - - #Resistence filter - for r in s.resistances: - if (s.ST > 0 or 'Novel' in s.ST_status ) and (r.identity >= config["threshold"]["motif_id"] and \ - r.span >= (config["threshold"]["motif_span"]/100.0)) or (s.ST < 0 and not 'Novel' in s.ST_status): - r.threshold = 'Passed' + sample_info = gen_add_info(sample_info) + + def gen_add_info(sample_info=dict()): + """ Enhances a sample info struct by adding ST_status, threshold info, versioning and sorting """ + #Set ST status + for s in sample_info: + s.ST_status=str(s.ST) + if s.Customer_ID_sample.startswith('NTC') or s.Customer_ID_sample.startswith('0-') or \ + s.Customer_ID_sample.startswith('NK-') or s.Customer_ID_sample.startswith('NEG') or \ + s.Customer_ID_sample.startswith('CTRL') or s.Customer_ID_sample.startswith('Neg') or \ + s.Customer_ID_sample.startswith('blank') or s.Customer_ID_sample.startswith('dual-NTC'): + s.ST_status = 'Kontroll (prefix)' + + if 'Kontroll' in s.ST_status or 'Control' in s.ST_status or s.ST == -1: + s.threshold = '-' + elif s.ST == -3: + s.threshold = 'Failed' + elif hasattr(s, 'seq_types') and s.seq_types != [] or s.ST == -2: + near_hits=0 + s.threshold = 'Passed' + for seq_type in s.seq_types: + #Identify single deviating allele + if seq_type.st_predictor and seq_type.identity >= config["threshold"]["mlst_novel_id"] and \ + config["threshold"]["mlst_id"] > seq_type.identity and 1-abs(1-seq_type.span) >= (config["threshold"]["mlst_span"]/100.0): + near_hits = near_hits + 1 + elif (seq_type.identity < config["threshold"]["mlst_novel_id"] or \ + seq_type.span < (config["threshold"]["mlst_span"]/100.0)) and seq_type.st_predictor: + s.threshold = 'Failed' + + if near_hits > 0 and s.threshold == 'Passed': + s.ST_status = 'Unknown ({} alleles)'.format(near_hits) else: - r.threshold = 'Failed' - - #Seq_type and resistance sorting - s.seq_types=sorted(s.seq_types, key=lambda x: x.loci) - s.resistances=sorted(s.resistances, key=lambda x: x.instance) - output['samples'].append(s) - - versions = session.query(Versions).all() - for version in versions: - name = version.name[8:] - output['versions'][name] = version.version - - return output + s.threshold = 'Failed' + + if not ('Control' in s.ST_status or 'Kontroll' in s.ST_status) and s.ST < 0: + if s.ST == -1: + s.ST_status = 'Invalid data' + elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Passed': + s.ST_status = 'Novel' + elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Failed': + s.ST_status = 'Unknown' + else: + s.ST_status='None' + + #Resistence filter + for r in s.resistances: + if (s.ST > 0 or 'Novel' in s.ST_status ) and (r.identity >= config["threshold"]["motif_id"] and \ + r.span >= (config["threshold"]["motif_span"]/100.0)) or (s.ST < 0 and not 'Novel' in s.ST_status): + r.threshold = 'Passed' + else: + r.threshold = 'Failed' + + #Seq_type and resistance sorting + s.seq_types=sorted(s.seq_types, key=lambda x: x.loci) + s.resistances=sorted(s.resistances, key=lambda x: x.instance) + output['samples'].append(s) + + versions = session.query(Versions).all() + for version in versions: + name = version.name[8:] + output['versions'][name] = version.version + + return output diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index b6415506..7a09b01c 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -11,7 +11,7 @@ # maintain the same connection per thread from sqlalchemy.pool import SingletonThreadPool -from microSALT.store.orm_models import app, Projects, Reports, Resistances, Samples, Seq_types, Steps, Versions +from microSALT.store.orm_models import app, Collections, Projects, Reports, Resistances, Samples, Seq_types, Steps, Versions from microSALT.store.models import Profiles, Novel class DB_Manipulator: @@ -53,6 +53,9 @@ def create_tables(self): if not self.engine.dialect.has_table(self.engine, 'reports'): Reports.__table__.create(self.engine) self.logger.info("Created reports table") + if not self.engine.dialect.has_table(self.engine, 'collections'): + Collections.__table__.create(self.engine) + self.logger.info("Created collections table") for k,v in self.profiles.items(): if not self.engine.dialect.has_table(self.engine, "profile_{}".format(k)): self.profiles[k].create() @@ -124,16 +127,17 @@ def upd_rec(self, req_dict, tablename, upd_dict): def purge_rec(self, name, type): """Removes seq_data, resistances, sample(s) and possibly project""" entries = list() - if type == "project": + if type == "Projects": entries.append(self.session.query(Projects).filter(Projects.CG_ID_project==name).all()) entries.append(self.session.query(Seq_types).filter(Seq_types.CG_ID_sample.like('{}%'.format(name))).all()) entries.append(self.session.query(Resistances).filter(Resistances.CG_ID_sample.like('{}%'.format(name))).all()) entries.append(self.session.query(Samples).filter(Samples.CG_ID_sample.like('{}%'.format(name))).all()) - elif type == "sample": + elif type == "Samples": entries.append(self.session.query(Seq_types).filter(Seq_types.CG_ID_sample==name).all()) entries.append(self.session.query(Resistances).filter(Resistances.CG_ID_sample==name).all()) entries.append(self.session.query(Samples).filter(Samples.CG_ID_sample==name).all()) - pass + elif type == "Collections": + entries.append(self.session.query(Collections).filter(Collections.ID_collection==name).all()) else: self.logger.error("Incorrect type {} specified for removal of {}. Check code".format(type, name)) sys.exit() diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 4b32f970..f9861671 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -21,11 +21,7 @@ def load_lims_project_info(self, cg_projid): samplelist = self.samples_in_project(cg_projid) custids = list() - for sample in samplelist: - self.load_lims_sample_info(sample) - custids.append(self.data['Customer_ID']) - if not custids[1:] == custids[:-1]: - raise Exception("Project {} contains multiple Customer IDs".format(cg_projid)) + self.load_lims_sample_info(samplelist[0]) try: #Resolves old format if ' ' in project.name: @@ -38,7 +34,7 @@ def load_lims_project_info(self, cg_projid): self.data.update({'date_received': newdate, 'CG_ID_project': cg_projid, 'Customer_ID_project' : realname, - 'Customer_ID': custids[0]}) + 'Customer_ID': self.data['Customer_ID']}) except KeyError as e: self.logger.warn("Unable to fetch LIMS info for project {}\nSource: {}".format(cg_projid, str(e))) @@ -54,27 +50,30 @@ def samples_in_project(self, cg_projid): output.append(s.id) return output - def load_lims_sample_info(self, cg_sampleid): + def load_lims_sample_info(self, cg_sampleid, warnings=False): """ Loads all utilized LIMS info. Organism assumed to be written as binomial name """ libprep_date = "" seq_date = "" - try: - #Internal - sample = Sample(self.lims, id=cg_sampleid) #External - if not sample: + if self.lims.get_samples(name=cg_sampleid): sample = self.lims.get_samples(name=cg_sampleid) if len(sample) != 1: - self.logger.error("Sample ID {} resolves to multiple entries".format(cg_sampleid)) + errnames = list() + for s in sample: + errnames.append(s.id) + if warnings: + self.logger.warn("Sample name {} resolves to entries '{}'. Arbitarily picking {}".format(s.name, (', '.join(errnames)), sample[0].id )) sample = sample[0] - + #Internal + else: + sample = Sample(self.lims, id=cg_sampleid) method_libprep = self.get_method(cg_sampleid,type='libprep') method_sequencing = self.get_method(cg_sampleid,type='sequencing') date_libprep = self.get_date(cg_sampleid,type="libprep") date_sequencing = self.get_date(cg_sampleid,type="sequencing") except Exception as e: - self.logger.error("LIMS connection timeout") + self.logger.error("LIMS connection timeout: '{}'".format(str(e))) organism = "Unset" if 'Strain' in sample.udf and organism == "Unset": @@ -180,8 +179,11 @@ def get_date(self, sample_id, type=""): date_list = date_list + [a.parent_process.date_run for a in arts] except Exception as e: pass - dp = max(date_list).split('-') - return datetime(int(dp[0]), int(dp[1]), int(dp[2])) + if date_list: + dp = max(date_list).split('-') + return datetime(int(dp[0]), int(dp[1]), int(dp[2])) + else: + return datetime.min def get_method(self, sample_id, type=""): """Retrives method document name and version for a sample""" @@ -203,7 +205,11 @@ def get_method(self, sample_id, type=""): processes = [(a.parent_process.udf[key_values['method']], a.parent_process.udf[key_values['version']]) for a in arts] processes = list(set(processes)) process = sorted(processes)[-1] - out = "{}:{}".format(process[0], process[1]) + if processes: + process = sorted(processes)[-1] + out = "{}:{}".format(process[0], process[1]) except Exception as e: pass + if not 'out' in locals(): + out = "Not in LIMS" return out diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index db248375..ffa65a5e 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -294,6 +294,10 @@ def create_snpsection(self): def create_project(self, name): """Creates project in database""" + if self.pool: + self.db_pusher.purge_rec(self.name, 'Collections') + for sample in self.pool: + self.db_pusher.add_rec({'collection_ID':self.name, 'CG_ID_sample':sample}, 'Collections') try: self.lims_fetcher.load_lims_project_info(name) except Exception as e: @@ -341,8 +345,8 @@ def project_job(self, single_sample=False): self.create_project(os.path.normpath(self.indir).split('/')[-2]) elif self.pool: addedprojs = list() - for sample in pool: - proj = sample[:-3] + for sample in self.pool: + proj = re.search('(\w+)A(?:\w+)', sample).group(1) if proj not in addedprojs: self.create_project(proj) addedprojs.append(proj) @@ -418,7 +422,9 @@ def finish_job(self, joblist, single_sample=False): report = 'default' scope = 'project' if single_sample: - scope = 'sample' + scope = 'sample' + elif self.pool: + scope = 'collection' if self.qc_only: report = 'qc' diff --git a/microSALT/utils/reporter.py b/microSALT/utils/reporter.py index 02077cf3..85f185bf 100644 --- a/microSALT/utils/reporter.py +++ b/microSALT/utils/reporter.py @@ -20,14 +20,15 @@ from multiprocessing import Process -from microSALT.server.views import app, session, gen_reportdata +from microSALT.server.views import app, session, gen_reportdata, gen_collectiondata from microSALT.store.orm_models import Samples from microSALT.store.lims_fetcher import LIMS_Fetcher class Reporter(): - def __init__(self, config, log, name = "", output = ""): + def __init__(self, config, log, name = "", output = "", collection=False): self.name = name + self.collection = collection if output == "": self.output = os.getcwd() else: @@ -109,21 +110,25 @@ def gen_qc(self,silent=False): self.error = True def gen_typing(self,silent=False): - try: - self.ticketFinder.load_lims_project_info(self.name) - except Exception as e: - self.logger.error("Project {} does not exist".format(self.name)) - self.kill_flask() - sys.exit(-1) - try: - r = requests.get("http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), allow_redirects=True) + if len(self.name) == 1: + try: + self.ticketFinder.load_lims_project_info(self.name) + except Exception as e: + self.logger.error("Project {} does not exist".format(self.name)) + self.kill_flask() + sys.exit(-1) outfile = "{}/{}_Typing_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) + r = requests.get("http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), allow_redirects=True) + else: + outfile = "{}/{}-{}_Typing_{}.html".format(self.output, "collection", self.name, self.now) + r = requests.get("http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), allow_redirects=True) + try: output ="{}/{}".format(self.output, outfile) storage = "{}/{}".format(self.config['folders']['reports'], outfile) open(output, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) copyfile(primary, storage) - + self.filelist.append(output) self.filelist.append(storage) if not silent: @@ -133,10 +138,13 @@ def gen_typing(self,silent=False): self.error = True def gen_resistence(self, silent=False): - self.ticketFinder.load_lims_project_info(self.name) + if self.collection: + sample_info = gen_collectiondata(self.name) + else: + self.ticketFinder.load_lims_project_info(self.name) + sample_info = gen_reportdata(self.name) output = "{}/{}_{}.csv".format(self.output,self.name,self.now) excel = open(output, "w+") - sample_info = gen_reportdata(self.name) resdict = dict() #Load ALL resistance & genes diff --git a/microSALT/utils/scraper.py b/microSALT/utils/scraper.py index 4304d06f..e50f5e62 100644 --- a/microSALT/utils/scraper.py +++ b/microSALT/utils/scraper.py @@ -24,9 +24,11 @@ def __init__(self, infolder, config, log): self.infolder = os.path.abspath(infolder) self.sampledir = "" - last_folder = os.path.basename(os.path.normpath(self.infolder)) + last_folder = self.infolder.split('/')[-1] self.name = last_folder.split('_')[0] #TODO: Replace date from dir with entry from analysis files/database + if not '_' in last_folder: + last_folder = self.infolder.split('/')[-2] self.date = "{} {}".format(re.sub('\.','-', last_folder.split('_')[1]), re.sub('\.',':', last_folder.split('_')[2])) self.db_pusher=DB_Manipulator(config, log) self.lims_fetcher=LIMS_Fetcher(config, log) From 3e085a075f2627321bf9f30d5c5c7f329e8206b7 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 11:03:11 +0200 Subject: [PATCH 51/83] Forgot to push orm update --- microSALT/store/orm_models.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index 4686e9be..15f25305 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -15,6 +15,7 @@ class Samples(db.Model): seq_types = relationship("Seq_types", back_populates="samples") projects = relationship('Projects', back_populates='samples') resistances = relationship("Resistances", back_populates="samples") + steps = relationship("Steps", back_populates="samples") CG_ID_sample = db.Column(db.String(15), primary_key=True, nullable=False) CG_ID_project = db.Column(db.String(15), ForeignKey('projects.CG_ID_project')) @@ -90,7 +91,7 @@ class Resistances(db.Model): #Multi-date support for libprep/sequencing/analysis class Steps(db.Model): __tablename__ = 'steps' - samples= relationship("Samples", back_populates="steps") + samples = relationship("Samples", back_populates="steps") CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) step = db.Column(db.String(40)) @@ -100,6 +101,7 @@ class Steps(db.Model): class Projects(db.Model): __tablename__ = 'projects' samples = relationship('Samples', back_populates='projects') + reports = relationship('Reports', back_populates='projects') CG_ID_project = db.Column(db.String(15), primary_key=True, nullable=False) Customer_ID_project = db.Column(db.String(15)) @@ -121,3 +123,9 @@ class Reports(db.Model): steps_aggregate = db.Column(db.String(100)) date = db.Column(db.DateTime) version = db.Column(db.Integer, default=1) + +class Collections(db.Model): + __tablename__ = 'collections' + + ID_collection = db.Column(db.String(15), primary_key=True) + CG_ID_sample = db.Column(db.String(15)) From 71d01f206b1017e7d5a9d2a08bcc8005ecbb7592 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 11:16:34 +0200 Subject: [PATCH 52/83] Syntax error --- microSALT/server/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/microSALT/server/views.py b/microSALT/server/views.py index d429843c..b436c11b 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -115,7 +115,6 @@ def gen_reportdata(pid='all', organism_group='all'): sample_info = sorted(sample_info, key=lambda sample: \ int(sample.CG_ID_sample.replace(sample.CG_ID_project, '')[1:])) -<<<<<<< HEAD sample_info = gen_add_info(sample_info) def gen_add_info(sample_info=dict()): From b3e70418fff7c6a0b8dec9172674fc0af1c61293 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 13:43:45 +0200 Subject: [PATCH 53/83] Removed empty method description from QC page --- microSALT/server/templates/alignment_page.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index ce8d4d36..acdb4dcf 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -137,11 +137,6 @@

Provsammanställning

-

Analysbegränsningar för: {{samples[0].application_tag}}

-

- - -

-

Avvikelser från metoden

All kommunikation gällande ordern såsom tillägg, avvikelser eller From 9a7ae3f3a140ebefcbc76279ca7f8e88af06bd9e Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 13:52:08 +0200 Subject: [PATCH 54/83] Upgraded verified organism check --- configExample.json | 4 +++- microSALT/server/templates/typing_page.html | 6 ++---- microSALT/server/views.py | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/configExample.json b/configExample.json index d51e724e..1a0b8824 100644 --- a/configExample.json +++ b/configExample.json @@ -12,7 +12,9 @@ "_comment": "File finding patterns. Only single capture group accepted (for reverse/forward identifier)", "regex": { "mail_recipient": "user@examplegroup.com", - "file_pattern": "\\w{8,12}_\\w{8,10}(?:-\\d+)*_L\\d_(?:R)*(\\d{1}).fastq.gz" + "file_pattern": "\\w{8,12}_\\w{8,10}(?:-\\d+)*_L\\d_(?:R)*(\\d{1}).fastq.gz", + "_comment": "Organisms recognized enough to be considered stable", + "verified_organisms": ["Enterococcus faecalis","Enterococcus faecium","Escherichia coli","Klebsiella pneumoniae","Staphylococcus aureus"] }, "_comment": "Folders", diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 3c3e5ca4..c5b8acc8 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -100,7 +100,7 @@

Provsammanställning

 {% endif %} - {% if sample.priority == 'standard' %} + {% if sample.priority == 'standard' and sample.organism.replace('_', ' ').capitalize() in verified_organisms %} {% else %} @@ -208,11 +208,9 @@

Signatur för godkännande av rapport

Detaljresultat - {% if sample.priority == 'standard' %} + {% if sample.priority == 'standard' and sample.organism.replace('_', ' ').capitalize() in verified_organisms %} Swedac Logo - {% else %} -

Icke verifierad organism

{% endif %}
Klinisk Mikrobiologi diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 0d6da614..e973441f 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -68,6 +68,7 @@ def typing_page(project, organism_group): date = date.today().isoformat(), version = sample_info['versions'], threshold = config['threshold'], + verified_organisms = config['regex']['verified_organisms'] build = __version__) From 33280fa75b15bb719da07a14c4ee439d6ab555d3 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 15:44:22 +0200 Subject: [PATCH 55/83] Analysis collection start, attempt 3 --- microSALT/cli.py | 2 +- microSALT/server/views.py | 120 ++++++++++++++++++--------------- microSALT/store/orm_models.py | 2 +- microSALT/utils/job_creator.py | 37 +++++----- microSALT/utils/scraper.py | 10 ++- 5 files changed, 94 insertions(+), 77 deletions(-) diff --git a/microSALT/cli.py b/microSALT/cli.py index 4eef6f97..088fba6f 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -360,7 +360,7 @@ def project(ctx, project_id, rerun, email, input, config, report): @click.option('--email', default=config['regex']['mail_recipient'], help='Forced e-mail recipient') @click.option('--input', help='Full path to result sample folder', default="") @click.option('--config', help="microSALT config to override default", default="") -@click.option('--report', default='default', type=click.Choice(['default', 'qc'])) +@click.option('--report', default='default', type=click.Choice(['default', 'qc','resistance_overview'])) @click.pass_context def collection(ctx, collection_id, rerun, email, input, config, report): """Parse results from analysing a set of sample""" diff --git a/microSALT/server/views.py b/microSALT/server/views.py index b436c11b..4c159a4b 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -86,21 +86,18 @@ def STtracker_page(customer): return render_template('STtracker_page.html', internal = final_samples) -def gen_collectiondata(samples=[]): +def gen_collectiondata(collect_id=[]): """ Queries database using a set of samples""" - session.query(Collections).filter(Collections.ID_collection==samples) - print("TO BE IMPLEMENTED") - - #prefix_sam = ["CG_ID_sample=="+ x for x in samples] - #sample_info = session.query(Samples).filter(or_(','.join(prefix_sam))) - - #sample_info = gen_add_info(sample_info) + arglist = [] + samples = session.query(Collections).filter(Collections.ID_collection==collect_id).all() + for sample in samples: + arglist.append("Samples.CG_ID_sample=='{}'".format(sample.CG_ID_sample)) + sample_info = session.query(Samples).filter(eval("or_({})".format(','.join(arglist)))) + sample_info = gen_add_info(sample_info) + return sample_info def gen_reportdata(pid='all', organism_group='all'): """ Queries database for all necessary information for the reports """ - output = dict() - output['samples'] = list() - output['versions'] = dict() if pid=='all' and organism_group=='all': sample_info = session.query(Samples) elif pid=='all': @@ -111,59 +108,74 @@ def gen_reportdata(pid='all', organism_group='all'): sample_info = session.query(Samples).\ filter(Samples.CG_ID_project==pid, Samples.organism==organism_group) + sample_info = gen_add_info(sample_info) + return sample_info + +def gen_add_info(sample_info=dict()): + """ Enhances a sample info struct by adding ST_status, threshold info, versioning and sorting """ + #Set ST status + output = dict() + output['samples'] = list() + output['versions'] = dict() + #Sorts sample names sample_info = sorted(sample_info, key=lambda sample: \ int(sample.CG_ID_sample.replace(sample.CG_ID_project, '')[1:])) - sample_info = gen_add_info(sample_info) - - def gen_add_info(sample_info=dict()): - """ Enhances a sample info struct by adding ST_status, threshold info, versioning and sorting """ - #Set ST status - for s in sample_info: - s.ST_status=str(s.ST) - if s.Customer_ID_sample.startswith('NTC') or s.Customer_ID_sample.startswith('0-') or \ - s.Customer_ID_sample.startswith('NK-') or s.Customer_ID_sample.startswith('NEG') or \ - s.Customer_ID_sample.startswith('CTRL') or s.Customer_ID_sample.startswith('Neg') or \ - s.Customer_ID_sample.startswith('blank') or s.Customer_ID_sample.startswith('dual-NTC'): - s.ST_status = 'Kontroll (prefix)' - - if 'Kontroll' in s.ST_status or 'Control' in s.ST_status or s.ST == -1: - s.threshold = '-' - elif s.ST == -3: - s.threshold = 'Failed' - elif hasattr(s, 'seq_types') and s.seq_types != [] or s.ST == -2: - near_hits=0 - s.threshold = 'Passed' - for seq_type in s.seq_types: - #Identify single deviating allele - if seq_type.st_predictor and seq_type.identity >= config["threshold"]["mlst_novel_id"] and \ - config["threshold"]["mlst_id"] > seq_type.identity and 1-abs(1-seq_type.span) >= (config["threshold"]["mlst_span"]): - near_hits = near_hits + 1 - elif (seq_type.identity < config["threshold"]["mlst_novel_id"] or \ - seq_type.span < (config["threshold"]["mlst_span"])) and seq_type.st_predictor: - s.threshold = 'Failed' - - if near_hits > 0 and s.threshold == 'Passed': - s.ST_status = 'Unknown ({} alleles)'.format(near_hits) + for s in sample_info: + s.ST_status=str(s.ST) + if s.Customer_ID_sample.startswith('NTC') or s.Customer_ID_sample.startswith('0-') or \ + s.Customer_ID_sample.startswith('NK-') or s.Customer_ID_sample.startswith('NEG') or \ + s.Customer_ID_sample.startswith('CTRL') or s.Customer_ID_sample.startswith('Neg') or \ + s.Customer_ID_sample.startswith('blank') or s.Customer_ID_sample.startswith('dual-NTC'): + s.ST_status = 'Kontroll (prefix)' + + if 'Kontroll' in s.ST_status or 'Control' in s.ST_status or s.ST == -1: + s.threshold = '-' + elif s.ST == -3: + s.threshold = 'Failed' + elif hasattr(s, 'seq_types') and s.seq_types != [] or s.ST == -2: + near_hits=0 + s.threshold = 'Passed' + for seq_type in s.seq_types: + #Identify single deviating allele + if seq_type.st_predictor and seq_type.identity >= config["threshold"]["mlst_novel_id"] and \ + config["threshold"]["mlst_id"] > seq_type.identity and 1-abs(1-seq_type.span) >= (config["threshold"]["mlst_span"]/100.0): + near_hits = near_hits + 1 + elif (seq_type.identity < config["threshold"]["mlst_novel_id"] or \ + seq_type.span < (config["threshold"]["mlst_span"]/100.0)) and seq_type.st_predictor: + s.threshold = 'Failed' + + if near_hits > 0 and s.threshold == 'Passed': + s.ST_status = 'Unknown ({} alleles)'.format(near_hits) + else: s.threshold = 'Failed' + if not ('Control' in s.ST_status or 'Kontroll' in s.ST_status) and s.ST < 0: + if s.ST == -1: + s.ST_status = 'Data saknas' + elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Passed': + s.ST_status = 'Novel' + elif (s.ST <= -4 or s.ST == -2) and s.threshold == 'Failed': + s.ST_status = 'Unknown' + else: + s.ST_status='None' + #Resistence filter for r in s.resistances: - if (s.ST > 0 or 'Novel' in s.ST_status ) and (r.identity >= config["threshold"]["motif_id"] and \ - r.span >= config["threshold"]["motif_span"]) or (s.ST < 0 and not 'Novel' in s.ST_status): + if (r.identity >= config["threshold"]["motif_id"] and r.span >= config["threshold"]["motif_span"]/100.0): r.threshold = 'Passed' else: - s.threshold = 'Failed' + r.threshold = 'Failed' - #Seq_type and resistance sorting - s.seq_types=sorted(s.seq_types, key=lambda x: x.loci) - s.resistances=sorted(s.resistances, key=lambda x: x.instance) - output['samples'].append(s) + #Seq_type and resistance sorting + s.seq_types=sorted(s.seq_types, key=lambda x: x.loci) + s.resistances=sorted(s.resistances, key=lambda x: x.instance) + output['samples'].append(s) - versions = session.query(Versions).all() - for version in versions: - name = version.name[8:] - output['versions'][name] = version.version + versions = session.query(Versions).all() + for version in versions: + name = version.name[8:] + output['versions'][name] = version.version - return output + return output diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index 15f25305..234243ff 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -128,4 +128,4 @@ class Collections(db.Model): __tablename__ = 'collections' ID_collection = db.Column(db.String(15), primary_key=True) - CG_ID_sample = db.Column(db.String(15)) + CG_ID_sample = db.Column(db.String(15), primary_key=True) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 44e1123f..bceb726c 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -291,13 +291,22 @@ def create_snpsection(self): batchfile.write('\n') batchfile.close() + def create_collection(self): + """Creates collection entry in database""" + if self.db_pusher.exists('Collections', {'ID_collection':self.name}): + self.db_pusher.purge_rec(name=self.name, type='Collections') + for sample in self.pool: + self.db_pusher.add_rec({'ID_collection':self.name, 'CG_ID_sample':sample}, 'Collections') + + addedprojs = list() + for sample in self.pool: + proj = re.search('(\w+)A(?:\w+)', sample).group(1) + if proj not in addedprojs: + self.create_project(proj) + addedprojs.append(proj) def create_project(self, name): """Creates project in database""" - if self.pool: - self.db_pusher.purge_rec(self.name, 'Collections') - for sample in self.pool: - self.db_pusher.add_rec({'collection_ID':self.name, 'CG_ID_sample':sample}, 'Collections') try: self.lims_fetcher.load_lims_project_info(name) except Exception as e: @@ -344,12 +353,7 @@ def project_job(self, single_sample=False): if single_sample: self.create_project(os.path.normpath(self.indir).split('/')[-2]) elif self.pool: - addedprojs = list() - for sample in self.pool: - proj = re.search('(\w+)A(?:\w+)', sample).group(1) - if proj not in addedprojs: - self.create_project(proj) - addedprojs.append(proj) + self.create_collection() else: self.create_project(self.name) except Exception as e: @@ -425,8 +429,12 @@ def finish_job(self, joblist, single_sample=False): scope = 'sample' elif self.pool: scope = 'collection' + report = 'resistance_overview' if self.qc_only: report = 'qc' + custom_conf = '' + if 'config_path' in self.config: + custom_conf = '--config {}'.format(self.config['config_path']) startfile = "{}/run_started.out".format(self.finishdir) @@ -441,15 +449,8 @@ def finish_job(self, joblist, single_sample=False): mb.write("export MICROSALT_CONFIG={}\n".format(os.environ['MICROSALT_CONFIG'])) mb.write("source activate $CONDA_DEFAULT_ENV\n") - span = 'project' - custom_conf = '' - if single_sample: - span = 'sample' - if 'config_path' in self.config: - custom_conf = '--config {}'.format(self.config['config_path']) - mb.write("microSALT utils finish {} {} --input {} --rerun --email {} --report {} {}\n".\ - format(span, self.name, self.finishdir, self.config['regex']['mail_recipient'], report, custom_conf)) + format(scope, self.name, self.finishdir, self.config['regex']['mail_recipient'], report, custom_conf)) mb.write("touch {}/run_complete.out".format(self.finishdir)) mb.close() diff --git a/microSALT/utils/scraper.py b/microSALT/utils/scraper.py index e50f5e62..e50280d0 100644 --- a/microSALT/utils/scraper.py +++ b/microSALT/utils/scraper.py @@ -21,18 +21,22 @@ class Scraper(): def __init__(self, infolder, config, log): self.config = config self.logger = log + self.db_pusher=DB_Manipulator(config, log) + self.lims_fetcher=LIMS_Fetcher(config, log) + self.job_fallback=Job_Creator("", config, log) self.infolder = os.path.abspath(infolder) self.sampledir = "" last_folder = self.infolder.split('/')[-1] self.name = last_folder.split('_')[0] + #Cruddy internal check + if not 'ACC' in self.name and not 'MIC' in self.name: + self.lims_fetcher.load_lims_sample_info(self.name) + self.name = self.lims_fetcher.data['CG_ID_sample'] #TODO: Replace date from dir with entry from analysis files/database if not '_' in last_folder: last_folder = self.infolder.split('/')[-2] self.date = "{} {}".format(re.sub('\.','-', last_folder.split('_')[1]), re.sub('\.',':', last_folder.split('_')[2])) - self.db_pusher=DB_Manipulator(config, log) - self.lims_fetcher=LIMS_Fetcher(config, log) - self.job_fallback=Job_Creator("", config, log) self.lims_sample_info = {} self.gene2resistance = self.load_resistances() From 2a2646c68aa5fdbcca5d64deaf56e3ac7b9ffded Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 15:45:50 +0200 Subject: [PATCH 56/83] Changed sequencing date to finished date (instead of entered in lims) --- microSALT/store/lims_fetcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 79a93031..ecf101dd 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -176,7 +176,7 @@ def get_date(self, sample_id, type=""): for step in steps: try: arts = self.lims.get_artifacts(samplelimsid = sample_id, process_type = step) - date_list = date_list + [a.parent_process.date_run for a in arts] + date_list = date_list + [a.parent_process.udf['Finish Date'] for a in arts] except Exception as e: pass if date_list: From 51fdd8e2b05914f72fd8f29751b44a0a2aab6da4 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 15:55:47 +0200 Subject: [PATCH 57/83] Replaced creation date with arrival date --- microSALT/server/templates/typing_page.html | 6 +++++- microSALT/store/lims_fetcher.py | 13 +++++++++++-- microSALT/store/orm_models.py | 1 + microSALT/utils/job_creator.py | 1 + 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index c5b8acc8..c28ae57a 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -78,7 +78,11 @@

Provsammanställning

{% else %}

{% endif %} - + {% if sample.date_arrival is not none %} + + {% else %} + + {% endif %} {% if sample.date_libprep is not none %} {% else %} diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index ecf101dd..9fe3cc70 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -64,6 +64,7 @@ def load_lims_sample_info(self, cg_sampleid, external=False): sample = sample[0] method_libprep = self.get_method(cg_sampleid,type='libprep') method_sequencing = self.get_method(cg_sampleid,type='sequencing') + date_arrival = self.get_date(cg_sampleid,type="arrival") date_libprep = self.get_date(cg_sampleid,type="libprep") date_sequencing = self.get_date(cg_sampleid,type="sequencing") else: @@ -71,6 +72,7 @@ def load_lims_sample_info(self, cg_sampleid, external=False): sample = Sample(self.lims, id=cg_sampleid) method_libprep = self.get_method(cg_sampleid,type='libprep') method_sequencing = self.get_method(cg_sampleid,type='sequencing') + date_arrival = self.get_date(cg_sampleid,type="arrival") date_libprep = self.get_date(cg_sampleid,type="libprep") date_sequencing = self.get_date(cg_sampleid,type="sequencing") except Exception as e: @@ -167,7 +169,9 @@ def get_organism_refname(self, sample_name, external=False): def get_date(self, sample_id, type=""): """ Returns the most recent sequencing date of a sample """ date_list = list() - if type == "sequencing": + if type == "arrival": + steps = ["CG002 - Reception Control"] + elif type == "sequencing": steps = ["CG002 - Illumina Sequencing (Illumina SBS)", "CG002 Illumina SBS (HiSeq X)"] elif type == "libprep": steps = ["CG002 - Aggregate QC (Library Validation)"] @@ -176,7 +180,12 @@ def get_date(self, sample_id, type=""): for step in steps: try: arts = self.lims.get_artifacts(samplelimsid = sample_id, process_type = step) - date_list = date_list + [a.parent_process.udf['Finish Date'] for a in arts] + if type == "arrival": + date_list = date_list + [a.parent_process.udf['date arrived at clinical genomics'] for a in arts] + elif type == "sequencing": + date_list = date_list + [a.parent_process.udf['Finish Date'] for a in arts] + elif type == "libprep": + date_list = date_list + [a.parent_process.date_run for a in arts] except Exception as e: pass if date_list: diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index 3c15e874..2fd8b90f 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -41,6 +41,7 @@ class Samples(db.Model): reference_genome = db.Column(db.String(32)) application_tag = db.Column(db.String(15)) + date_arrival = db.Column(db.DateTime) date_analysis = db.Column(db.DateTime) date_sequencing = db.Column(db.DateTime) date_libprep = db.Column(db.DateTime) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index b693a735..d770a739 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -311,6 +311,7 @@ def create_sample(self, name): sample_col['organism']=self.lims_fetcher.data['organism'] sample_col["application_tag"] = self.lims_fetcher.data['application_tag'] sample_col["priority"] = self.lims_fetcher.data['priority'] + sample_col["date_arrival"] = self.lims_fetcher.data['date_arrival'] sample_col["date_sequencing"] = self.lims_fetcher.data['date_sequencing'] sample_col["date_libprep"] = self.lims_fetcher.data['date_libprep'] sample_col["method_libprep"] = self.lims_fetcher.data['method_libprep'] From 5d840e689f5cae3991a6993136a8ba9a7d8009e4 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 16:08:06 +0200 Subject: [PATCH 58/83] Cleaned up asterix notes in typing report --- microSALT/server/templates/typing_page.html | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index c28ae57a..d8c5bdc7 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -66,8 +66,8 @@

Provsammanställning

 - - + + {% for sample in samples %} @@ -114,10 +114,7 @@

Provsammanställning

# LociN/A{{samples[0].projects.date_ordered.date()}}{{sample.date_libprep.date()}}Okänd{{sample.date_libprep.date()}}{{sample.method_libprep}}{{sample.date_sequencing.date()}}Okänd{{sample.date_sequencing.date()}}{{sample.method_sequencing}} {{sample.priority.capitalize()}}{{sample.date_libprep.date()}}{{sample.method_libprep}}Okänd{{sample.date_sequencing.date()}}N/A{{samples[0].projects.date_ordered.date()}}OkändOkänt{{sample.date_libprep.date()}}{{sample.method_libprep}}OkändOkänt{{sample.method_libprep}}Okänt{{sample.date_sequencing.date()}}{{sample.method_sequencing}}Okänt{{sample.method_sequencing}}{{sample.priority.capitalize()}}✔︎N/A{{samples[0].projects.date_ordered.date()}}Okänt{{sample.date_libprep.date()}}Okänt{{sample.method_libprep}}Okänt{{sample.date_sequencing.date()}}{{sample.method_libprep}}Okänt{{sample.date_sequencing.date()}}Okänt{{sample.method_sequencing}}Okänt{{sample.priority.capitalize()}}N/A{{samples[0].projects.date_ordered.date()}}{{sample.date_libprep.date()}}Okänt{{sample.method_libprep}}Okänt{{sample.date_sequencing.date()}}Okänt{{sample.method_sequencing}}OkäntOkänt{{sample.priority.capitalize()}}✔︎- N/A{{samples[0].projects.date_ordered.date()}}{{sample.date_arrival.date()}}Okänt{{sample.date_libprep.date()}}Metod prep Datum sekvensering Metod sekvenseringPrioritet (*)(**)VerifieradPrioritet (*)Verifierad (**)
(*) Information om prover som kommer från kund
- (**) Standard = Verifierad organim, - Research = Icke verifierad organism
- Kvalitetskontrollen har verifierats för ett antal bestämda organismer. - För andra organismer kan ej kriterierna för metoden appliceras. + (**) Provet har processerats enligt standardförfarande, och består av en väl studerad organism
@@ -125,10 +122,8 @@

Provsammanställning

-

Resultatsammanställning(***)

+

Resultatsammanställning

- (***) Laboratoriet har inte haft ansvar för provtagningsstadiet och - extraktion, resultaten gäller för provet såsom det har mottagits.

@@ -136,7 +131,7 @@

Resultatsammanställning(***)

- + {% for sample in samples %} @@ -166,6 +161,8 @@

Resultatsammanställning(***)

{% endfor %}
Prov ID(*) CG Prov ID Organism (*)SekvenstypSekvenstyp (**) Tröskelvärden
+ (*) Information om prover som kommer från kund
+ (**) För förbättrad översikt rapporteras resultaten för kontrollprover enbart i detaljrapporten
@@ -181,6 +178,7 @@

Teknisk beskrivning av analysen : {{samples[0].application_tag}}Begränsningar av analysen : {{samples[0].application_tag}}

Analysen kan enbart beställas av gruppen Klinisk Mikrobiologi. + Laboratoriet har inte haft ansvar för provtagningsstadiet och extraktion, resultaten gäller för provet såsom det har mottagits. Typningen utgår från de publikt tillgängliga databaserna pubMLST och resFinder. Typningen av prover är enbart att anses som tillförlitlig om desamma uppnår de fördefinierade tröskelvärdena. Uppnåelse av tröskelvärdena garanteras i sin tur enbart för prover vars förhantering och detaljer tidigare verifierats av personal på Clinical Genomics. From 7d1b68c459286b825c94fa8da994d454b990273d Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 5 Jul 2019 17:21:06 +0200 Subject: [PATCH 59/83] Removed rerun flag from collection due to issues --- microSALT/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/cli.py b/microSALT/cli.py index 088fba6f..410a3dee 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -373,7 +373,7 @@ def collection(ctx, collection_id, rerun, email, input, config, report): except Exception as e: pass - ctx.obj['config']['rerun'] = rerun + ctx.obj['config']['rerun'] = False ctx.obj['config']['regex']['mail_recipient'] = email pool = [] From 372fb6dfd01bb86447a738ff3aa7924b225d0e41 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Sun, 7 Jul 2019 13:31:50 +0200 Subject: [PATCH 60/83] Experimental collection updates --- microSALT/cli.py | 1 - microSALT/store/lims_fetcher.py | 12 ++++++++++-- microSALT/utils/job_creator.py | 10 ++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/microSALT/cli.py b/microSALT/cli.py index 410a3dee..dd0f5c9e 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -235,7 +235,6 @@ def collection(ctx, collection_id, input, dry, qc_only, config, email, untrimmed pool_cg.append(scientist.data['CG_ID_sample']) except Exception as e: click.echo("Unable to load LIMS sample info for sample {}.".format(sample)) - sys.exit(-1) click.echo("Checking versions of references..") for sample in pool_cg: diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 84a35181..92c768ea 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -56,15 +56,23 @@ def load_lims_sample_info(self, cg_sampleid, warnings=False): seq_date = "" try: #External + num = 0 if self.lims.get_samples(name=cg_sampleid): sample = self.lims.get_samples(name=cg_sampleid) if len(sample) != 1: + prio = ['MIC1474', 'MIC2201', 'ACC4805'] errnames = list() for s in sample: errnames.append(s.id) + for p in prio: + for s in sample: + if p in s.id: + num=sample.index(s) + break if warnings: - self.logger.warn("Sample name {} resolves to entries '{}'. Arbitarily picking {}".format(s.name, (', '.join(errnames)), sample[0].id )) - sample = sample[0] + self.logger.warn("Sample name {} resolves to entries '{}'. Arbitarily picking {}".format(s.name, (', '.join(errnames)), sample[num].id )) + sample = sample[num] + #Internal else: sample = Sample(self.lims, id=cg_sampleid) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index bceb726c..458e2e58 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -114,7 +114,7 @@ def create_assemblysection(self): else: careline = '' - batchfile.write("spades.py --threads {} {} --memory {} -o {}/assembly -1 {} -2 {} {}\n"\ + batchfile.write("spades.py -k 81 --threads {} {} --memory {} -o {}/assembly -1 {} -2 {} {}\n"\ .format(self.config["slurm_header"]["threads"], careline, 8*int(self.config["slurm_header"]["threads"]), self.finishdir, self.concat_files['f'], self.concat_files['r'], trimline)) batchfile.write("rm {} {}\n".format(self.concat_files['f'], self.concat_files['r'])) batchfile.write("\n\n") @@ -425,11 +425,13 @@ def finish_job(self, joblist, single_sample=False): """ Uploads data and sends an email once all analysis jobs are complete. """ report = 'default' scope = 'project' + extraflag = '--rerun' if single_sample: scope = 'sample' elif self.pool: scope = 'collection' report = 'resistance_overview' + extraflag = '--collection' if self.qc_only: report = 'qc' custom_conf = '' @@ -449,8 +451,8 @@ def finish_job(self, joblist, single_sample=False): mb.write("export MICROSALT_CONFIG={}\n".format(os.environ['MICROSALT_CONFIG'])) mb.write("source activate $CONDA_DEFAULT_ENV\n") - mb.write("microSALT utils finish {} {} --input {} --rerun --email {} --report {} {}\n".\ - format(scope, self.name, self.finishdir, self.config['regex']['mail_recipient'], report, custom_conf)) + mb.write("microSALT utils finish {} {} --input {} {} --email {} --report {} {}\n".\ + format(scope, self.name, self.finishdir, extraflag, self.config['regex']['mail_recipient'], report, custom_conf)) mb.write("touch {}/run_complete.out".format(self.finishdir)) mb.close() @@ -480,7 +482,7 @@ def finish_job(self, joblist, single_sample=False): final = entry break - head = "-A {} -p core -n 1 -t 06:00:00 -J {}_{}_MAILJOB --qos {} --open-mode append --dependency=afterany:{} --output {}"\ + head = "-A {} -p core -n 1 -t 16:00:00 -J {}_{}_MAILJOB --qos {} --open-mode append --dependency=afterany:{} --output {}"\ .format(self.config["slurm_header"]["project"],self.config["slurm_header"]["job_prefix"],\ self.name,self.config["slurm_header"]["qos"],\ final, self.config['folders']['log_file'], self.config['regex']['mail_recipient']) From 85b1c5fe81e3685cd73c8cf2719c903c4c666f23 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Sun, 7 Jul 2019 13:37:29 +0200 Subject: [PATCH 61/83] Various collection fixes --- microSALT/cli.py | 1 - microSALT/store/lims_fetcher.py | 3 ++- microSALT/utils/job_creator.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/microSALT/cli.py b/microSALT/cli.py index dd0f5c9e..85c613c7 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -159,7 +159,6 @@ def sample(ctx, sample_id, input, dry, config, email, qc_only, untrimmed, skip_u scientist.load_lims_sample_info(sample_id) except Exception as e: click.echo("Unable to load LIMS sample info.") - sys.exit(-1) if input != "": sample_dir = os.path.abspath(input) diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 92c768ea..1e6d6f90 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -60,7 +60,8 @@ def load_lims_sample_info(self, cg_sampleid, warnings=False): if self.lims.get_samples(name=cg_sampleid): sample = self.lims.get_samples(name=cg_sampleid) if len(sample) != 1: - prio = ['MIC1474', 'MIC2201', 'ACC4805'] + #External priority list + prio = [] errnames = list() for s in sample: errnames.append(s.id) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 458e2e58..6c3c4755 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -114,7 +114,7 @@ def create_assemblysection(self): else: careline = '' - batchfile.write("spades.py -k 81 --threads {} {} --memory {} -o {}/assembly -1 {} -2 {} {}\n"\ + batchfile.write("spades.py --threads {} {} --memory {} -o {}/assembly -1 {} -2 {} {}\n"\ .format(self.config["slurm_header"]["threads"], careline, 8*int(self.config["slurm_header"]["threads"]), self.finishdir, self.concat_files['f'], self.concat_files['r'], trimline)) batchfile.write("rm {} {}\n".format(self.concat_files['f'], self.concat_files['r'])) batchfile.write("\n\n") @@ -482,7 +482,7 @@ def finish_job(self, joblist, single_sample=False): final = entry break - head = "-A {} -p core -n 1 -t 16:00:00 -J {}_{}_MAILJOB --qos {} --open-mode append --dependency=afterany:{} --output {}"\ + head = "-A {} -p core -n 1 -t 06:00:00 -J {}_{}_MAILJOB --qos {} --open-mode append --dependency=afterany:{} --output {}"\ .format(self.config["slurm_header"]["project"],self.config["slurm_header"]["job_prefix"],\ self.name,self.config["slurm_header"]["qos"],\ final, self.config['folders']['log_file'], self.config['regex']['mail_recipient']) From ef8267b1051d59cb6c9f94896695a69342606054 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Sun, 7 Jul 2019 20:22:19 +0200 Subject: [PATCH 62/83] Minifix --- microSALT/utils/job_creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 6c3c4755..d45a58c5 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -431,7 +431,7 @@ def finish_job(self, joblist, single_sample=False): elif self.pool: scope = 'collection' report = 'resistance_overview' - extraflag = '--collection' + extraflag = '' if self.qc_only: report = 'qc' custom_conf = '' From e0f8b27b73a7a45fb3d8ce05588a27508e7cdcb2 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Mon, 8 Jul 2019 15:12:37 +0200 Subject: [PATCH 63/83] Multi-date storage deemed redundant by EoL and thus backrolled --- microSALT/store/db_manipulator.py | 5 +---- microSALT/store/orm_models.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 7a09b01c..9c876b48 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -11,7 +11,7 @@ # maintain the same connection per thread from sqlalchemy.pool import SingletonThreadPool -from microSALT.store.orm_models import app, Collections, Projects, Reports, Resistances, Samples, Seq_types, Steps, Versions +from microSALT.store.orm_models import app, Collections, Projects, Reports, Resistances, Samples, Seq_types, Versions from microSALT.store.models import Profiles, Novel class DB_Manipulator: @@ -47,9 +47,6 @@ def create_tables(self): if not self.engine.dialect.has_table(self.engine, 'resistances'): Resistances.__table__.create(self.engine) self.logger.info("Created resistance table") - if not self.engine.dialect.has_table(self.engine, 'steps'): - Steps.__table__.create(self.engine) - self.logger.info("Created step table") if not self.engine.dialect.has_table(self.engine, 'reports'): Reports.__table__.create(self.engine) self.logger.info("Created reports table") diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index 234243ff..dee3bcb6 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -89,14 +89,14 @@ class Resistances(db.Model): contig_end=db.Column(db.Integer) #Multi-date support for libprep/sequencing/analysis -class Steps(db.Model): - __tablename__ = 'steps' - samples = relationship("Samples", back_populates="steps") - - CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) - step = db.Column(db.String(40)) - method = db.Column(db.String(40)) - date = db.Column(db.DateTime) +#class Steps(db.Model): +# __tablename__ = 'steps' +# samples = relationship("Samples", back_populates="steps") +# +# CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) +# step = db.Column(db.String(40), primary_key=True) +# method = db.Column(db.String(40), primary_key=True) +# date = db.Column(db.DateTime) class Projects(db.Model): __tablename__ = 'projects' From b856dbaa86d91825dfa560517f1e3e384c1cee33 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 12:52:46 +0200 Subject: [PATCH 64/83] Automatic report numbering, attempt 1 --- microSALT/server/templates/typing_page.html | 7 +++++ microSALT/server/views.py | 6 ++++ microSALT/store/db_manipulator.py | 31 +++++++++++++++++++++ microSALT/store/orm_models.py | 2 +- microSALT/utils/reporter.py | 8 ++++++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 493c11f1..408ad43a 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -45,6 +45,13 @@

Projektsammanställning

Kund ID (*) {{samples[0].projects.Customer_ID}} + + Rapport version + {% if len(reports) == 1 %} + Version {{reports[0].version}}, {{reports[0].date.date()}} + {% len(reports) > 1 %} + Version {{reports[0].version}}, {{reports[0].date.date()}} (Ersätter version {{reports[0].version}}, {{reports[0].date.date()}}) + microSALT version {{build}} diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 4c159a4b..8e4fbb62 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -57,6 +57,7 @@ def alignment_page(project): date = date.today().isoformat(), version = sample_info['versions'], threshold = config['threshold'], + reports = sample_info['reports'], build = __version__) @app.route('/microSALT//typing/') @@ -68,6 +69,7 @@ def typing_page(project, organism_group): date = date.today().isoformat(), version = sample_info['versions'], threshold = config['threshold'], + reports = sample_info['reports'], build = __version__) @@ -109,6 +111,10 @@ def gen_reportdata(pid='all', organism_group='all'): filter(Samples.CG_ID_project==pid, Samples.organism==organism_group) sample_info = gen_add_info(sample_info) + + reports = session.query(Reports).filter(Reports.CG_ID_project==pid).all() + sample_info['reports'] = reports=sorted(reports, key=lambda x: x.version reverse=True) + return sample_info def gen_add_info(sample_info=dict()): diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 9c876b48..bc4fc735 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -3,14 +3,17 @@ #!/usr/bin/env python +import hashlib import sys import warnings +from datetime import datetime from sqlalchemy import * from sqlalchemy.orm import sessionmaker # maintain the same connection per thread from sqlalchemy.pool import SingletonThreadPool +from microSALT import __version__ from microSALT.store.orm_models import app, Collections, Projects, Reports, Resistances, Samples, Seq_types, Versions from microSALT.store.models import Profiles, Novel @@ -218,6 +221,34 @@ def get_version(self, name): else: return version.version + def get_report(self, name): + #Sort based on version + prev_reports = self.sessions.query(Reports).filter(Reports.CG_ID_project==name).order_by(desc(Reports.version).all() + if len(prev_reports) > 0: + prev_report = prev_reports[0] + return prev_report + + def set_report(self, name): + #Generate string + totalstring = list() + dt = datetime.now() + samples = self.sessions.query(Samples).filter(Samples.CG_ID_project==name).order_by(desc(Samples.CG_ID_sample).all() + for sample in samples: + totalstring.append(sample.date_libprep) + totalstring.append(sample.method_libprep) + totalstring.append(sample.date_sequencing) + totalstring.append(sample.method_sequencing) + totalstring.append(__version__) + totalstring = ''.join(totalstring).encode() + hashstring = hashlib.md5(totalstring).hexdigest() + + prev_report = self.get_report(name) + #Compare + if 'steps_aggregate' in prev_report and prev_report.steps_aggregate != hashstring: + self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt 'version':prev_report.version+1} ,'Reports') + else: + self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt 'version':prev_report.version+1} ,'Reports') + def add_external(self,overwrite=False, sample=""): """Looks at each novel table. See if any record has a profile match in the profile table. Updates these based on parameters""" diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index dee3bcb6..011bf3fb 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -122,7 +122,7 @@ class Reports(db.Model): CG_ID_project = db.Column(db.String(15), ForeignKey('projects.CG_ID_project'), primary_key=True) steps_aggregate = db.Column(db.String(100)) date = db.Column(db.DateTime) - version = db.Column(db.Integer, default=1) + version = db.Column(db.Integer, default=1, primary_key=True) class Collections(db.Model): __tablename__ = 'collections' diff --git a/microSALT/utils/reporter.py b/microSALT/utils/reporter.py index 85f185bf..08a7aa88 100644 --- a/microSALT/utils/reporter.py +++ b/microSALT/utils/reporter.py @@ -20,13 +20,16 @@ from multiprocessing import Process +from microSALT import __version__ from microSALT.server.views import app, session, gen_reportdata, gen_collectiondata +from microSALT.store.db_manipulator import DB_Manipulator from microSALT.store.orm_models import Samples from microSALT.store.lims_fetcher import LIMS_Fetcher class Reporter(): def __init__(self, config, log, name = "", output = "", collection=False): + self.db_pusher=DB_Manipulator(config, log) self.name = name self.collection = collection if output == "": @@ -47,6 +50,7 @@ def __init__(self, config, log, name = "", output = "", collection=False): format(self.dt.year, self.dt.month, self.dt.day, self.dt.hour, self.dt.minute, self.dt.second)) def report(self, type='default', customer='all'): + self.gen_version(self.name) self.start_web() if type == 'json_dump': self.gen_json() @@ -71,6 +75,10 @@ def report(self, type='default', customer='all'): for file in self.filelist: os.remove(file) + def gen_version(self, name): + self.db_pusher.get_report(name) + self.db_pusher.set_report(name) + def gen_STtracker(self, customer="all", silent=False): self.name ="Sequence Type Update" try: From ba168d2090c9b5301d4271c7e187b69006a95b5a Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 13:06:57 +0200 Subject: [PATCH 65/83] Bugfix, added to alignment page --- microSALT/server/templates/alignment_page.html | 7 +++++++ microSALT/store/db_manipulator.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 5812e900..d0f05569 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -46,6 +46,13 @@

Projektsammanställning

Kund ID (*) {{samples[0].projects.Customer_ID}} + + Rapport version + {% if len(reports) == 1 %} + Version {{reports[0].version}}, {{reports[0].date.date()}} + {% len(reports) > 1 %} + Version {{reports[0].version}}, {{reports[0].date.date()}} (Ersätter version {{reports[0].version}}, {{reports[0].date.date()}}) + microSALT version {{build}} diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index bc4fc735..95c65c9c 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -223,7 +223,7 @@ def get_version(self, name): def get_report(self, name): #Sort based on version - prev_reports = self.sessions.query(Reports).filter(Reports.CG_ID_project==name).order_by(desc(Reports.version).all() + prev_reports = self.sessions.query(Reports).filter(Reports.CG_ID_project==name).order_by(desc(Reports.version)i).all() if len(prev_reports) > 0: prev_report = prev_reports[0] return prev_report @@ -232,7 +232,7 @@ def set_report(self, name): #Generate string totalstring = list() dt = datetime.now() - samples = self.sessions.query(Samples).filter(Samples.CG_ID_project==name).order_by(desc(Samples.CG_ID_sample).all() + samples = self.sessions.query(Samples).filter(Samples.CG_ID_project==name).order_by(desc(Samples.CG_ID_sample)).all() for sample in samples: totalstring.append(sample.date_libprep) totalstring.append(sample.method_libprep) From 775b2978bac3de7b8a11808340a8915951076fbf Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 13:12:41 +0200 Subject: [PATCH 66/83] Syntax error --- microSALT/server/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/server/views.py b/microSALT/server/views.py index e973441f..8991ab2b 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -68,7 +68,7 @@ def typing_page(project, organism_group): date = date.today().isoformat(), version = sample_info['versions'], threshold = config['threshold'], - verified_organisms = config['regex']['verified_organisms'] + verified_organisms = config['regex']['verified_organisms'], build = __version__) From 7a057e2b285f77ddce53de2edc20e8145a6f7bc5 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 13:25:42 +0200 Subject: [PATCH 67/83] Syntax fixed the syntax fix --- microSALT/store/db_manipulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 95c65c9c..6b79dd5f 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -223,7 +223,7 @@ def get_version(self, name): def get_report(self, name): #Sort based on version - prev_reports = self.sessions.query(Reports).filter(Reports.CG_ID_project==name).order_by(desc(Reports.version)i).all() + prev_reports = self.sessions.query(Reports).filter(Reports.CG_ID_project==name).order_by(desc(Reports.version)).all() if len(prev_reports) > 0: prev_report = prev_reports[0] return prev_report From fcff3e48eea1689f50ab71822eefc8cdcf43fbef Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 14:18:13 +0200 Subject: [PATCH 68/83] Added some commas --- microSALT/server/views.py | 2 +- microSALT/store/db_manipulator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 30042e68..41403e7f 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -114,7 +114,7 @@ def gen_reportdata(pid='all', organism_group='all'): sample_info = gen_add_info(sample_info) reports = session.query(Reports).filter(Reports.CG_ID_project==pid).all() - sample_info['reports'] = reports=sorted(reports, key=lambda x: x.version reverse=True) + sample_info['reports'] = reports=sorted(reports, key=lambda x: x.version, reverse=True) return sample_info diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 6b79dd5f..a0ca71fe 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -245,9 +245,9 @@ def set_report(self, name): prev_report = self.get_report(name) #Compare if 'steps_aggregate' in prev_report and prev_report.steps_aggregate != hashstring: - self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt 'version':prev_report.version+1} ,'Reports') + self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt, 'version':prev_report.version+1} ,'Reports') else: - self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt 'version':prev_report.version+1} ,'Reports') + self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt, 'version':prev_report.version+1} ,'Reports') def add_external(self,overwrite=False, sample=""): """Looks at each novel table. See if any record has a profile match in the profile table. From f9dc7fdf3502760e1d2362210157cdf877e20027 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 15:11:52 +0200 Subject: [PATCH 69/83] Improved config check error message --- microSALT/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/microSALT/__init__.py b/microSALT/__init__.py index abbf4552..037d345a 100644 --- a/microSALT/__init__.py +++ b/microSALT/__init__.py @@ -21,20 +21,20 @@ with open(envvar, 'r') as conf: config = json.load(conf) except Exception as e: - print("Config error: {}".format(e.args)) + print("Config error: {}".format(str(e))) pass elif os.path.exists(defaulto): try: with open(defaulto, 'r') as conf: config = json.load(conf) except Exception as e: - print("Config error: {}".format(e.args)) + print("Config error: {}".format(str(e))) pass # Load flask instance if config != '': try: app.config.update(config['database']) except Exception as e: - print("Config error: {}".format(e.args)) + print("Config error: {}".format(str(e))) pass From 2fedc3cc88cf6ff4f43b03f88f8ea693e1be909e Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 15:17:22 +0200 Subject: [PATCH 70/83] Removed steps reference --- microSALT/store/orm_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index 2d60133e..6a03956d 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -15,7 +15,7 @@ class Samples(db.Model): seq_types = relationship("Seq_types", back_populates="samples") projects = relationship('Projects', back_populates='samples') resistances = relationship("Resistances", back_populates="samples") - steps = relationship("Steps", back_populates="samples") + #steps = relationship("Steps", back_populates="samples") CG_ID_sample = db.Column(db.String(15), primary_key=True, nullable=False) CG_ID_project = db.Column(db.String(15), ForeignKey('projects.CG_ID_project')) From b50ab413bfeac70d8d917f7054bbc3d1fbffa4d3 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 15:36:57 +0200 Subject: [PATCH 71/83] LIMS datefix w legacy support --- microSALT/store/lims_fetcher.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 48de5882..5f10e9de 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -197,8 +197,11 @@ def get_date(self, sample_id, type=""): except Exception as e: pass if date_list: - dp = max(date_list).split('-') - return datetime(int(dp[0]), int(dp[1]), int(dp[2])) + try: + dp = max(date_list).split('-') + return datetime(int(dp[0]), int(dp[1]), int(dp[2])) + except Exception as e: + return max(date_list) else: return datetime.min From f8fabf5545c04c2358d0f9529079af1cbdee7cc5 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 15:59:08 +0200 Subject: [PATCH 72/83] LIMS arrival_date db upload --- microSALT/store/lims_fetcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 5f10e9de..5bd1c55e 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -140,6 +140,7 @@ def load_lims_sample_info(self, cg_sampleid, warnings=False): 'reference' : sample.udf['Reference Genome Microbial'], 'Customer_ID': sample.udf['customer'], 'application_tag': sample.udf['Sequencing Analysis'], + 'date_arrival': date_arrival, 'date_sequencing': date_sequencing, 'date_libprep': date_libprep, 'method_libprep': method_libprep, From aa4f605789857272f1b651af2029fa7bc64a2aa1 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Wed, 10 Jul 2019 17:06:46 +0200 Subject: [PATCH 73/83] Fixed various integration issues --- .../server/templates/alignment_page.html | 7 +++-- microSALT/server/templates/typing_page.html | 7 +++-- microSALT/server/views.py | 2 +- microSALT/store/db_manipulator.py | 16 +++++----- microSALT/utils/reporter.py | 30 ++++++++----------- microSALT/utils/scraper.py | 4 +-- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index d0f05569..e747b381 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -48,10 +48,11 @@

Projektsammanställning

Rapport version - {% if len(reports) == 1 %} - Version {{reports[0].version}}, {{reports[0].date.date()}} - {% len(reports) > 1 %} + {% if reports|length > 1 %} Version {{reports[0].version}}, {{reports[0].date.date()}} (Ersätter version {{reports[0].version}}, {{reports[0].date.date()}}) + {% else %} + Version {{reports[0].version}}, {{reports[0].date.date()}} + {% endif %} microSALT version diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 07d30b22..0cb34218 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -47,10 +47,11 @@

Projektsammanställning

Rapport version - {% if len(reports) == 1 %} - Version {{reports[0].version}}, {{reports[0].date.date()}} - {% len(reports) > 1 %} + {% if reports|length > 1 %} Version {{reports[0].version}}, {{reports[0].date.date()}} (Ersätter version {{reports[0].version}}, {{reports[0].date.date()}}) + {% else %} + Version {{reports[0].version}}, {{reports[0].date.date()}} + {% endif %} microSALT version diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 41403e7f..e4c506d6 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -10,7 +10,7 @@ from microSALT import config, __version__ from microSALT.store.db_manipulator import app -from microSALT.store.orm_models import Collections, Projects, Samples, Seq_types, Versions +from microSALT.store.orm_models import Collections, Projects, Reports, Samples, Seq_types, Versions engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], connect_args={'check_same_thread': False}) Session = sessionmaker(bind=engine) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index a0ca71fe..2b9c2cae 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -223,7 +223,8 @@ def get_version(self, name): def get_report(self, name): #Sort based on version - prev_reports = self.sessions.query(Reports).filter(Reports.CG_ID_project==name).order_by(desc(Reports.version)).all() + prev_report = [] + prev_reports = self.session.query(Reports).filter(Reports.CG_ID_project==name).order_by(desc(Reports.version)).all() if len(prev_reports) > 0: prev_report = prev_reports[0] return prev_report @@ -232,11 +233,11 @@ def set_report(self, name): #Generate string totalstring = list() dt = datetime.now() - samples = self.sessions.query(Samples).filter(Samples.CG_ID_project==name).order_by(desc(Samples.CG_ID_sample)).all() + samples = self.session.query(Samples).filter(Samples.CG_ID_project==name).order_by(desc(Samples.CG_ID_sample)).all() for sample in samples: - totalstring.append(sample.date_libprep) + totalstring.append(str(datetime.timestamp(sample.date_libprep))) totalstring.append(sample.method_libprep) - totalstring.append(sample.date_sequencing) + totalstring.append(str(datetime.timestamp(sample.date_sequencing))) totalstring.append(sample.method_sequencing) totalstring.append(__version__) totalstring = ''.join(totalstring).encode() @@ -244,10 +245,11 @@ def set_report(self, name): prev_report = self.get_report(name) #Compare - if 'steps_aggregate' in prev_report and prev_report.steps_aggregate != hashstring: - self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt, 'version':prev_report.version+1} ,'Reports') + if prev_report: + if 'steps_aggregate' in dir(prev_report) and prev_report.steps_aggregate != hashstring: + self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt, 'version':prev_report.version+1} ,'Reports') else: - self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt, 'version':prev_report.version+1} ,'Reports') + self.add_rec({'CG_ID_project':name, 'steps_aggregate':hashstring, 'date':dt, 'version':1} ,'Reports') def add_external(self,overwrite=False, sample=""): """Looks at each novel table. See if any record has a profile match in the profile table. diff --git a/microSALT/utils/reporter.py b/microSALT/utils/reporter.py index 08a7aa88..cfcfa0a5 100644 --- a/microSALT/utils/reporter.py +++ b/microSALT/utils/reporter.py @@ -102,12 +102,12 @@ def gen_qc(self,silent=False): sys.exit(-1) try: q = requests.get("http://127.0.0.1:5000/microSALT/{}/qc".format(self.name), allow_redirects=True) - outfile = "{}_QC_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) + outfile = "{}_QC_{}.html".format(self.ticketFinder.data['Customer_ID_project'], self.now) output ="{}/{}".format(self.output, outfile) storage = "{}/{}".format(self.config['folders']['reports'], outfile) - open(output, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) - copyfile(primary, storage) + open(output, 'wb').write(q.content.decode("iso-8859-1").encode("utf8")) + copyfile(output, storage) self.filelist.append(output) self.filelist.append(storage) @@ -118,24 +118,20 @@ def gen_qc(self,silent=False): self.error = True def gen_typing(self,silent=False): - if len(self.name) == 1: - try: - self.ticketFinder.load_lims_project_info(self.name) - except Exception as e: - self.logger.error("Project {} does not exist".format(self.name)) - self.kill_flask() - sys.exit(-1) - outfile = "{}/{}_Typing_{}.html".format(self.output, self.ticketFinder.data['Customer_ID_project'], self.now) - r = requests.get("http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), allow_redirects=True) - else: - outfile = "{}/{}-{}_Typing_{}.html".format(self.output, "collection", self.name, self.now) - r = requests.get("http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), allow_redirects=True) try: + self.ticketFinder.load_lims_project_info(self.name) + except Exception as e: + self.logger.error("Project {} does not exist".format(self.name)) + self.kill_flask() + sys.exit(-1) + try: + r = requests.get("http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), allow_redirects=True) + outfile = "{}_Typing_{}.html".format(self.ticketFinder.data['Customer_ID_project'], self.now) output ="{}/{}".format(self.output, outfile) storage = "{}/{}".format(self.config['folders']['reports'], outfile) open(output, 'wb').write(r.content.decode("iso-8859-1").encode("utf8")) - copyfile(primary, storage) + copyfile(output, storage) self.filelist.append(output) self.filelist.append(storage) @@ -246,7 +242,7 @@ def mail(self): if not self.error: msg['Subject'] = '{} ({}) Reports'.format(self.name, file_name[0].split('_')[0]) else: - msg['Subject'] = '{} ({}) Failed Generating Report'.format(self.name, file_name[0].split('_')[0]) + msg['Subject'] = '{} Failed Generating Report'.format(self.name) msg['From'] = 'microSALT@{}'.format(socket.gethostname()) msg['To'] = self.config['regex']['mail_recipient'] diff --git a/microSALT/utils/scraper.py b/microSALT/utils/scraper.py index e50280d0..e5260350 100644 --- a/microSALT/utils/scraper.py +++ b/microSALT/utils/scraper.py @@ -43,7 +43,7 @@ def __init__(self, infolder, config, log): def scrape_project(self): """Scrapes a project folder for information""" if self.config['rerun']: - self.db_pusher.purge_rec(self.name, 'project') + self.db_pusher.purge_rec(self.name, 'Projects') if not self.db_pusher.exists('Projects', {'CG_ID_project':self.name}): self.logger.error("Re-filling project {}".format(self.name)) self.job_fallback.create_project(self.name) @@ -64,7 +64,7 @@ def scrape_project(self): def scrape_sample(self): """Scrapes a sample folder for information""" if self.config['rerun']: - self.db_pusher.purge_rec(self.name, 'sample') + self.db_pusher.purge_rec(self.name, 'Samples') self.lims_fetcher.load_lims_sample_info(self.name) if not self.db_pusher.exists('Projects', {'CG_ID_project':self.lims_fetcher.data['CG_ID_project']}): From e4c6a491fb06f3a3870a6e9434b8362b5245c7e1 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 11 Jul 2019 11:06:04 +0200 Subject: [PATCH 74/83] More minor fixes --- microSALT/store/db_manipulator.py | 4 ++-- microSALT/store/lims_fetcher.py | 31 ++++++++++++++++++------------- microSALT/utils/scraper.py | 5 ++++- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 2b9c2cae..c040a4ef 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -235,9 +235,9 @@ def set_report(self, name): dt = datetime.now() samples = self.session.query(Samples).filter(Samples.CG_ID_project==name).order_by(desc(Samples.CG_ID_sample)).all() for sample in samples: - totalstring.append(str(datetime.timestamp(sample.date_libprep))) + totalstring.append(str(datetime.timestamp(sample.date_libprep)/1000.0)) totalstring.append(sample.method_libprep) - totalstring.append(str(datetime.timestamp(sample.date_sequencing))) + totalstring.append(str(datetime.timestamp(sample.date_sequencing)/1000.0)) totalstring.append(sample.method_sequencing) totalstring.append(__version__) totalstring = ''.join(totalstring).encode() diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 5bd1c55e..6005d1e7 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -85,7 +85,12 @@ def load_lims_sample_info(self, cg_sampleid, warnings=False): except Exception as e: self.logger.error("LIMS connection timeout: '{}'".format(str(e))) + #Figuring out the organism organism = "Unset" + reference = "None" + if 'Reference Genome Microbial' in sample.udf: + reference = sample.udf['Reference Genome Microbial'] + if 'Strain' in sample.udf and organism == "Unset": #Predefined genus usage. All hail buggy excel files if 'gonorrhoeae' in sample.udf['Strain']: @@ -94,12 +99,11 @@ def load_lims_sample_info(self, cg_sampleid, warnings=False): organism = "Propionibacterium acnes" #Backwards compat, MUST hit first elif sample.udf['Strain'] == 'VRE': - if 'Reference Genome Microbial' in sample.udf: - if sample.udf['Reference Genome Microbial'] == 'NC_017960.1': - organism = 'Enterococcus faecium' - elif sample.udf['Reference Genome Microbial'] == 'NC_004668.1': - organism = 'Enterococcus faecalis' - elif 'Comment' in sample.udf: + if reference == 'NC_017960.1': + organism = 'Enterococcus faecium' + elif reference == 'NC_004668.1': + organism = 'Enterococcus faecalis' + elif 'Comment' in sample.udf and not re.match('\w{4}\d{2,3}', sample.udf['Comment']): organism = sample.udf['Comment'] elif sample.udf['Strain'] != 'Other' and sample.udf['Strain'] != 'other': organism = sample.udf['Strain'] @@ -111,16 +115,16 @@ def load_lims_sample_info(self, cg_sampleid, warnings=False): organism = "Propionibacterium acnes" else: organism = sample.udf['Other species'] - if 'Reference Genome Microbial' in sample.udf and organism == "Unset": - if sample.udf['Reference Genome Microbial'] == 'NC_002163': + if reference != 'None' and organism == "Unset": + if reference == 'NC_002163': organism = "Campylobacter jejuni" - elif sample.udf['Reference Genome Microbial'] == 'NZ_CP007557.1': + elif reference == 'NZ_CP007557.1': organism = 'Klebsiella oxytoca' - elif sample.udf['Reference Genome Microbial'] == 'NC_000913.3': + elif reference == 'NC_000913.3': organism = 'Citrobacter freundii' - elif sample.udf['Reference Genome Microbial'] == 'NC_002516.2': + elif reference == 'NC_002516.2': organism = 'Pseudomonas aeruginosa' - elif 'Comment' in sample.udf and organism == "Unset": + elif 'Comment' in sample.udf and not re.match('\w{4}\d{2,3}', sample.udf['Comment']) and organism == "Unset": organism = sample.udf['Comment'] # Consistent safe-guard elif organism == "Unset": @@ -131,13 +135,14 @@ def load_lims_sample_info(self, cg_sampleid, warnings=False): prio = sample.udf['priority'] else: prio = "" + try: self.data.update({'CG_ID_project': sample.project.id, 'CG_ID_sample': sample.id, 'Customer_ID_sample' : sample.name, 'organism' : organism, 'priority' : prio, - 'reference' : sample.udf['Reference Genome Microbial'], + 'reference' : reference, 'Customer_ID': sample.udf['customer'], 'application_tag': sample.udf['Sequencing Analysis'], 'date_arrival': date_arrival, diff --git a/microSALT/utils/scraper.py b/microSALT/utils/scraper.py index e5260350..53dd803d 100644 --- a/microSALT/utils/scraper.py +++ b/microSALT/utils/scraper.py @@ -329,7 +329,10 @@ def scrape_alignment(self): for line in fh.readlines(): lsplit = line.rstrip().split('\t') if type == 'raw': - tot_reads = int(lsplit[0]) + try: + tot_reads = int(lsplit[0]) + except Exception as e: + pass elif type == 'ins': if len(lsplit) >= 18 and lsplit[-12] in ['FF','FR']: try: From bf84b63012ea1bda6f08ae2d4e77ac8a9cf7f4d5 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 11 Jul 2019 14:03:37 +0200 Subject: [PATCH 75/83] Fixed min date timestamping --- microSALT/store/db_manipulator.py | 4 ++-- microSALT/store/lims_fetcher.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index c040a4ef..2b9c2cae 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -235,9 +235,9 @@ def set_report(self, name): dt = datetime.now() samples = self.session.query(Samples).filter(Samples.CG_ID_project==name).order_by(desc(Samples.CG_ID_sample)).all() for sample in samples: - totalstring.append(str(datetime.timestamp(sample.date_libprep)/1000.0)) + totalstring.append(str(datetime.timestamp(sample.date_libprep))) totalstring.append(sample.method_libprep) - totalstring.append(str(datetime.timestamp(sample.date_sequencing)/1000.0)) + totalstring.append(str(datetime.timestamp(sample.date_sequencing))) totalstring.append(sample.method_sequencing) totalstring.append(__version__) totalstring = ''.join(totalstring).encode() diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index 6005d1e7..b3fad659 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -3,7 +3,7 @@ import os import re -from datetime import datetime +from datetime import datetime, timezone from genologics.lims import Lims # Should probably call these items directly since we're now up to 3 config files from genologics.entities import Project, Sample @@ -209,7 +209,7 @@ def get_date(self, sample_id, type=""): except Exception as e: return max(date_list) else: - return datetime.min + return datetime.min.replace(tzinfo=timezone.utc) def get_method(self, sample_id, type=""): """Retrives method document name and version for a sample""" From 83959ab561416721b8bdec96b55247d8a4cfb082 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 11 Jul 2019 14:46:22 +0200 Subject: [PATCH 76/83] Move timezone fix to db_manip --- microSALT/store/db_manipulator.py | 6 +++--- microSALT/store/lims_fetcher.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 2b9c2cae..6379d610 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -7,7 +7,7 @@ import sys import warnings -from datetime import datetime +from datetime import datetime, timezone from sqlalchemy import * from sqlalchemy.orm import sessionmaker # maintain the same connection per thread @@ -235,9 +235,9 @@ def set_report(self, name): dt = datetime.now() samples = self.session.query(Samples).filter(Samples.CG_ID_project==name).order_by(desc(Samples.CG_ID_sample)).all() for sample in samples: - totalstring.append(str(datetime.timestamp(sample.date_libprep))) + totalstring.append(str(datetime.timestamp(sample.date_libprep.replace(tzinfo=timezone.utc)))) totalstring.append(sample.method_libprep) - totalstring.append(str(datetime.timestamp(sample.date_sequencing))) + totalstring.append(str(datetime.timestamp(sample.date_sequencing.replace(tzinfo=timezone.utc))) totalstring.append(sample.method_sequencing) totalstring.append(__version__) totalstring = ''.join(totalstring).encode() diff --git a/microSALT/store/lims_fetcher.py b/microSALT/store/lims_fetcher.py index b3fad659..6005d1e7 100644 --- a/microSALT/store/lims_fetcher.py +++ b/microSALT/store/lims_fetcher.py @@ -3,7 +3,7 @@ import os import re -from datetime import datetime, timezone +from datetime import datetime from genologics.lims import Lims # Should probably call these items directly since we're now up to 3 config files from genologics.entities import Project, Sample @@ -209,7 +209,7 @@ def get_date(self, sample_id, type=""): except Exception as e: return max(date_list) else: - return datetime.min.replace(tzinfo=timezone.utc) + return datetime.min def get_method(self, sample_id, type=""): """Retrives method document name and version for a sample""" From 9a82f6b4c4f893ef204ae72c19ae30d299822adc Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 11 Jul 2019 14:53:03 +0200 Subject: [PATCH 77/83] Syntax error --- microSALT/store/db_manipulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index 6379d610..f9588165 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -237,7 +237,7 @@ def set_report(self, name): for sample in samples: totalstring.append(str(datetime.timestamp(sample.date_libprep.replace(tzinfo=timezone.utc)))) totalstring.append(sample.method_libprep) - totalstring.append(str(datetime.timestamp(sample.date_sequencing.replace(tzinfo=timezone.utc))) + totalstring.append(str(datetime.timestamp(sample.date_sequencing.replace(tzinfo=timezone.utc)))) totalstring.append(sample.method_sequencing) totalstring.append(__version__) totalstring = ''.join(totalstring).encode() From 501dc2e617c513aaf148dc0beee2ef326ab8a472 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 11 Jul 2019 15:17:43 +0200 Subject: [PATCH 78/83] Extra organism check --- microSALT/server/templates/typing_page.html | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index 0cb34218..ebaae32c 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -111,10 +111,12 @@

Provsammanställning

Okänt {% endif %} {{sample.priority.capitalize()}} - {% if sample.priority == 'standard' and sample.organism.replace('_', ' ').capitalize() in verified_organisms %} - ✔︎ - {% else %} - - + {% if sample.organism is not none %} + {% if sample.priority == 'standard' and sample.organism.replace('_', ' ').capitalize() in verified_organisms %} + ✔︎ + {% else %} + - + {% endif %} {% endif %} {% endfor %} @@ -227,9 +229,11 @@

Signatur för godkännande av rapport

Detaljresultat - {% if sample.priority == 'standard' and sample.organism.replace('_', ' ').capitalize() in verified_organisms %} - Swedac Logo + {% if sample.organism is not none %} + {% if sample.priority == 'standard' and sample.organism.replace('_', ' ').capitalize() in verified_organisms %} + Swedac Logo + {% endif %} {% endif %}
Klinisk Mikrobiologi From 0510d5e9bed73e6cd1f5c2b46a80238dfe803daf Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Thu, 11 Jul 2019 15:21:16 +0200 Subject: [PATCH 79/83] Even more check --- microSALT/server/templates/typing_page.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index ebaae32c..b1379364 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -117,6 +117,8 @@

Provsammanställning

{% else %} - {% endif %} + {% else %} + - {% endif %} {% endfor %} From e8444191d2c409d4ce40783e068a629588d11f2e Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 12 Jul 2019 10:06:14 +0200 Subject: [PATCH 80/83] Added x to average coverage --- microSALT/server/templates/alignment_page.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index e747b381..6245a4f1 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -151,11 +151,11 @@

Provsammanställning

0.0x {% elif (sample.average_coverage)|round(2) <= threshold.average_coverage_warn and not 'Kontroll' in sample.ST_status %} {% if (sample.average_coverage)|round(2) <= threshold.average_coverage_fail %} - {{ (sample.average_coverage)|round(2) }} + {{ (sample.average_coverage)|round(2) }}x {% else %} - {{ (sample.average_coverage)|round(2) }} + {{ (sample.average_coverage)|round(2) }}x {% endif %} - {% else %} {{ (sample.average_coverage)|round(2) }}x {% endif %} + {% else %} {{ (sample.average_coverage)|round(2) }}x{% endif %} {% if sample.coverage_10x is none or (sample.coverage_10x*100)|round(2) == 0 %} 0.0 From 0a221f7da7f6a71929285634c4c73c926f623d33 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 12 Jul 2019 10:30:19 +0200 Subject: [PATCH 81/83] Swedified phone number --- microSALT/server/templates/alignment_page.html | 2 +- microSALT/server/templates/typing_page.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 6245a4f1..97df5710 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -23,7 +23,7 @@

Kontakt Clinical Genomics

Science for Life Laboratory
Tomtebodavägen 23
171 65 Solna
- P: (08) 524 81 500 + T: (08) 524 81 500

diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index b1379364..a2b5c977 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -22,7 +22,7 @@

Kontakt Clinical Genomics

Science for Life Laboratory
Tomtebodavägen 23
171 65 Solna
- P: (08) 524 81 500 + T: (08) 524 81 500
From 58778c878657e7e3660fc8c724390220e4e4f810 Mon Sep 17 00:00:00 2001 From: Isak Sylvin Date: Fri, 12 Jul 2019 10:41:24 +0200 Subject: [PATCH 82/83] Removed strong from headers --- microSALT/server/templates/alignment_page.html | 4 ++-- microSALT/server/templates/typing_page.html | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/microSALT/server/templates/alignment_page.html b/microSALT/server/templates/alignment_page.html index 97df5710..a7ee23b8 100644 --- a/microSALT/server/templates/alignment_page.html +++ b/microSALT/server/templates/alignment_page.html @@ -61,7 +61,7 @@

Projektsammanställning

-

Provsammanställning

+

Provsammanställning

@@ -194,7 +194,7 @@

Provsammanställning

-

Teknisk beskrivning av analysen: {{samples[0].application_tag}}

+

Teknisk beskrivning av analysen: {{samples[0].application_tag}}

{% if samples[0].application_tag == "MWRNXTR003" %} Mikrobiell helgenomssekvensering av rutinprov, med krav på minst 3 miljoner läspar. diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index a2b5c977..f80f3715 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -180,7 +180,7 @@

Resultatsammanställning

{% if samples[0].application_tag == "MWRNXTR003" %} -

Teknisk beskrivning av analysen: {{samples[0].application_tag}}

+

Teknisk beskrivning av analysen: {{samples[0].application_tag}}

{% if samples[0].application_tag == "MWRNXTR003" %} Mikrobiell helgenomssekvensering av rutinprov, med krav på minst 3 miljoner läspar. @@ -197,7 +197,7 @@

Teknisk beskrivning av analysen: {{samples[0].application_tag}}

-

Analysbegränsningar för: {{samples[0].application_tag}}

+

Analysbegränsningar för: {{samples[0].application_tag}}

Analysen kan enbart beställas av gruppen Klinisk Mikrobiologi. Laboratoriet har inte haft ansvar för provtagningsstadiet och extraktion, resultaten gäller för provet såsom det har mottagits. @@ -205,7 +205,7 @@

Analysbegränsningar för: {{samples[0].application_tag}} -

Avvikelser från metoden

+

Avvikelser från metoden

All kommunikation gällande order såsom tillägg, avvikelser eller ev undantag i metoden från Clinical Genomics finns tillgängligt i @@ -213,7 +213,7 @@

Avvikelser från metoden

öppnas upp igen för frågor.

-

Signatur för godkännande av rapport

+

Signatur för godkännande av rapport

Valtteri Wirta
Head of unit, Clinical Genomics @@ -230,7 +230,7 @@

Signatur för godkännande av rapport

-

Detaljresultat +

Detaljresultat {% if sample.organism is not none %} {% if sample.priority == 'standard' and sample.organism.replace('_', ' ').capitalize() in verified_organisms %} Swedac Logo Date: Fri, 12 Jul 2019 13:36:30 +0200 Subject: [PATCH 83/83] Added version, improved file size warning message --- microSALT/__init__.py | 2 +- microSALT/utils/job_creator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/microSALT/__init__.py b/microSALT/__init__.py index 037d345a..4b3d20d7 100644 --- a/microSALT/__init__.py +++ b/microSALT/__init__.py @@ -4,7 +4,7 @@ from flask import Flask -__version__ = '2.7.0' +__version__ = '2.8.0' app = Flask(__name__, template_folder='server/templates') app.config.setdefault('SQLALCHEMY_DATABASE_URI', 'sqlite:///:memory:') diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 6e66ab46..f8d842aa 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -98,7 +98,7 @@ def verify_fastq(self): bsize = os.stat("{}/{}".format(self.indir,vfile)).st_size bsize = bsize >> 20 if bsize > 1000: - self.logger.warning("Input fastq {} exceeds 1000MB") + self.logger.warning("Input fastq {} exceeds 1000MB".format(vfile)) return verified_files def create_assemblysection(self):