Merge pull request #64 from Clinical-Genomics/accreditation-changes-t…

…o-report

Accreditation changes to report

configExample.json

@@ -12,13 +12,17 @@
   "_comment": "File finding patterns. Only single capture group accepted (for reverse/forward identifier)",
   "regex":  {
     "mail_recipient": "[email protected]",
-    "file_pattern": "\\w{8,12}_\\w{8,10}(?:-\\d+)*_L\\d_(?:R)*(\\d{1}).fastq.gz"
+    "file_pattern": "\\w{8,12}_\\w{8,10}(?:-\\d+)*_L\\d_(?:R)*(\\d{1}).fastq.gz",
+    "_comment": "Organisms recognized enough to be considered stable",
+    "verified_organisms": ["Enterococcus faecalis","Enterococcus faecium","Escherichia coli","Klebsiella pneumoniae","Staphylococcus aureus"]
   },
 
   "_comment": "Folders",
   "folders":  {
     "_comment": "Root folder for ALL output",
     "results": "/mnt/hds/proj/bioinfo/MICROBIAL/MLST/results/",
+    "_comment": "Report collection folder",
+    "reports": "/mnt/hds/proj/bioinfo/MICROBIAL/MLST/reports/",
     "_comment": "Log file position and name",
     "log_file": "~/microsalt.log",
 
@@ -45,11 +49,30 @@
 
   "_comment": "Thresholds for Displayed results",
   "threshold":  {
+    "_comment": "Typing thresholds",
     "mlst_id": 100,
     "mlst_novel_id": 99.5,
-    "mlst_span": 0.9,
+    "mlst_span": 90,
     "motif_id": 97,
-    "motif_span": 0.9
+    "motif_span": 90,
+    "_comment": "Quality Control thresholds",
+    "total_reads_warn": 75,
+    "total_reads_fail": 70,
+    "NTC_total_reads_warn": 10,
+    "NTC_total_reads_fail": 20,
+    "mapped_rate_warn": 50,
+    "mapped_rate_fail": 30,
+    "duplication_rate_warn": 20,
+    "duplication_rate_fail": 80,
+    "insert_size_warn": 140,
+    "insert_size_fail": 100,
+    "average_coverage_warn": 100,
+    "average_coverage_fail": 10,
+    "bp_10x_warn": 90,
+    "bp_10x_fail": 80,
+    "bp_30x_warn": 70,
+    "bp_50x_warn": 50,
+    "bp_100x_warn": 20
   },
 
   "_comment": "Genologics temporary configuration file",

microSALT/__init__.py

@@ -4,7 +4,7 @@
 
 from flask import Flask
 
-__version__ = '2.7.1'
+__version__ = '2.8.0'
 
 app = Flask(__name__, template_folder='server/templates')
 app.config.setdefault('SQLALCHEMY_DATABASE_URI', 'sqlite:///:memory:')
@@ -21,20 +21,20 @@
     with open(envvar, 'r') as conf:
       config = json.load(conf)
   except Exception as e:
-    print("Config error: {}".format(e.args))
+    print("Config error: {}".format(str(e)))
     pass
 elif os.path.exists(defaulto):
   try:
     with open(defaulto, 'r') as conf:
       config = json.load(conf)
   except Exception as e:
-    print("Config error: {}".format(e.args)) 
+    print("Config error: {}".format(str(e))) 
     pass
 # Load flask instance
 if config != '':
   try:
     app.config.update(config['database'])
   except Exception as e:
-    print("Config error: {}".format(e.args))
+    print("Config error: {}".format(str(e)))
     pass
 

microSALT/cli.py

@@ -7,6 +7,8 @@
 import logging
 import json
 import os
+import re
+import subprocess
 import sys
 import yaml
 
@@ -22,6 +24,11 @@
 if config == '':
   click.echo("ERROR: No properly set-up config under neither envvar MICROSALT_CONFIG nor ~/.microSALT/config.json. Exiting.")
   sys.exit(-1)
+else:
+  #Makes sure DB inherits correct permissions if freshly created
+  bash_cmd="touch {}".format(re.search('sqlite\:\/\/\/(.+)', config['database']['SQLALCHEMY_DATABASE_URI']).group(1))
+  proc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE)
+  output, error = proc.communicate()
 
 def done():
   click.echo("Execution finished!")
@@ -152,7 +159,6 @@ def sample(ctx, sample_id, input, dry, config, email, qc_only, untrimmed, skip_u
     scientist.load_lims_sample_info(sample_id)
   except Exception as e:
     click.echo("Unable to load LIMS sample info.")
-    sys.exit(-1)
 
   if input != "":
     sample_dir = os.path.abspath(input)
@@ -180,6 +186,76 @@ def sample(ctx, sample_id, input, dry, config, email, qc_only, untrimmed, skip_u
     click.echo("Unable to process sample {} due to '{}'".format(sample_id,e))
   done()
 
+
+@analyse.command()
+@click.argument('collection_id')
+@click.option('--input', help='Full path to sample folder', default="")
+@click.option('--dry', help="Builds instance without posting to SLURM", default=False, is_flag=True)
+@click.option('--qc_only', help="Only runs QC (alignment stats)", default=False, is_flag=True)
+@click.option('--config', help="microSALT config to override default", default="")
+@click.option('--email', default=config['regex']['mail_recipient'], help='Forced e-mail recipient')
+@click.option('--untrimmed', help="Use untrimmed input data", default=False, is_flag=True)
+@click.option('--skip_update', default=False, help="Skips downloading of references", is_flag=True)
+@click.option('--uncareful', help="Avoids running SPAdes in careful mode. Sometimes fix assemblies", default=False, is_flag=True)
+@click.pass_context
+def collection(ctx, collection_id, input, dry, qc_only, config, email, untrimmed, skip_update, uncareful):
+  """Analyse a collection of samples"""
+  ctx.obj['config']['regex']['mail_recipient'] = email
+  trimmed = not untrimmed
+  careful = not uncareful
+  if config != '':
+    if os.path.exists(config):
+      try:
+        with open(os.path.abspath(config), 'r') as conf:
+          ctx.obj['config'] = json.load(conf)
+        ctx.obj['config']['config_path'] = os.path.abspath(config)
+      except Exception as e:
+        pass
+
+  ctx.obj['config']['dry'] = dry
+  scientist=LIMS_Fetcher(ctx.obj['config'], ctx.obj['log'])
+
+  pool = []
+  if input != "":
+    collection_dir = os.path.abspath(input)
+  else:
+    collection_dir = "{}/{}".format(ctx.obj['config']['folders']['seqdata'], collection_id)
+  for sample in os.listdir(collection_dir):
+    if os.path.isdir("{}/{}".format(collection_dir,sample)):
+      pool.append(sample)
+  if pool == []:
+    click.echo("Input collection lacks any valid of samples")
+    sys.exit(-1)
+
+  pool_cg = []
+  for sample in pool:
+    try:
+      scientist.load_lims_sample_info(sample,warnings=True)
+      pool_cg.append(scientist.data['CG_ID_sample'])
+    except Exception as e:
+      click.echo("Unable to load LIMS sample info for sample {}.".format(sample))
+
+  click.echo("Checking versions of references..")
+  for sample in pool_cg:
+    try:
+      if not skip_update:
+        fixer = Referencer(ctx.obj['config'], ctx.obj['log'])
+        fixer.identify_new(sample,project=False)
+    except Exception as e:
+      click.echo("Unable to update references for sample {} due to '{}'".format(sample,str(e)))
+  if not skip_update:
+    fixer.update_refs()
+    print("Version check done. Creating sbatch job")
+  else:
+    print("Skipping version check.")
+
+  try: 
+    worker = Job_Creator(collection_dir, ctx.obj['config'], ctx.obj['log'], trim=trimmed,qc_only=qc_only, careful=careful, pool=pool_cg)
+    worker.project_job()
+  except Exception as e:
+    click.echo("Unable to process collection due to '{}'".format(str(e)))
+  done()
+
 @finish.command()
 @click.argument('sample_id')
 @click.option('--rerun', is_flag=True, default=False, help='Overwrite existing data')
@@ -276,6 +352,67 @@ def project(ctx, project_id, rerun, email, input, config, report):
   codemonkey.report(report)
   done()
 
+@finish.command()
+@click.argument('collection_id')
+@click.option('--rerun', is_flag=True, default=False, help='Overwrite existing data')
+@click.option('--email', default=config['regex']['mail_recipient'], help='Forced e-mail recipient')
+@click.option('--input', help='Full path to result sample folder', default="")
+@click.option('--config', help="microSALT config to override default", default="")
+@click.option('--report', default='default', type=click.Choice(['default', 'qc','resistance_overview']))
+@click.pass_context
+def collection(ctx, collection_id, rerun, email, input, config, report):
+  """Parse results from analysing a set of sample"""
+  if config != '':
+    if os.path.exists(config):
+      try:
+        with open(os.path.abspath(config), 'r') as conf:
+          ctx.obj['config'] = json.load(conf)
+        ctx.obj['config']['config_path'] = os.path.abspath(config)
+      except Exception as e:
+        pass
+
+  ctx.obj['config']['rerun'] = False
+  ctx.obj['config']['regex']['mail_recipient'] = email
+
+  pool = []
+  if input != "":
+    collection_dir = os.path.abspath(input)
+  else:
+    prohits = [x for x in os.listdir(ctx.obj['config']['folders']['results']) if x.startswith("{}_".format(collection_id))]
+    if len(prohits) > 1:
+      click.echo("Multiple instances of that analysis exists. Specify full path using --input")
+      sys.exit(-1)
+    elif len(prohits) <1:
+      click.echo("No analysis folder prefixed by {} found.".format(project_id))
+      sys.exit(-1)
+    else:
+      collection_dir = "{}/{}".format(ctx.obj['config']['folders']['results'], prohits[-1])
+
+  for sample in os.listdir(collection_dir):
+    if os.path.isdir("{}/{}".format(collection_dir,sample)):
+      pool.append(sample)
+  if pool == []:
+    click.echo("Input collection lacks any valid of samples")
+    sys.exit(-1)
+
+  pool_cg = []
+  scientist=LIMS_Fetcher(ctx.obj['config'], ctx.obj['log'])
+  for sample in pool:
+    try:
+      scientist.load_lims_sample_info(sample,warnings=True)
+      pool_cg.append(scientist.data['CG_ID_sample'])
+    except Exception as e:
+      click.echo("Unable to load LIMS sample info for sample {}.".format(sample))
+      sys.exit(-1)
+
+  for sample in pool:
+    garbageman = Scraper("{}/{}".format(collection_dir, sample), ctx.obj['config'], ctx.obj['log'])
+    garbageman.scrape_sample()
+  codemonkey = Reporter(ctx.obj['config'], ctx.obj['log'], collection_id, output=collection_dir, collection=True)
+  codemonkey.report(report)
+  done()
+
+
 @refer.command()
 @click.argument('organism')
 @click.pass_context
@@ -306,11 +443,12 @@ def list(ctx):
 @click.option('--email', default=config['regex']['mail_recipient'], help='Forced e-mail recipient')
 @click.option('--type', default='default', type=click.Choice(['default', 'typing', 'resistance_overview', 'qc', 'json_dump', 'st_update']))
 @click.option('--output',help='Full path to output folder',default="")
+@click.option('--collection',default=False, is_flag=True)
 @click.pass_context
-def report(ctx, project_name, email, type, output):
+def report(ctx, project_name, email, type, output, collection):
   """Re-generates report for a project"""
   ctx.obj['config']['regex']['mail_recipient'] = email
-  codemonkey = Reporter(ctx.obj['config'], ctx.obj['log'], project_name, output)
+  codemonkey = Reporter(ctx.obj['config'], ctx.obj['log'], project_name, output, collection=collection)
   codemonkey.report(type)
   done()