diff --git a/cravat/base_converter.py b/cravat/base_converter.py index 3719707b..7674c7bf 100644 --- a/cravat/base_converter.py +++ b/cravat/base_converter.py @@ -1,3 +1,6 @@ +from cravat.config_loader import ConfigLoader + + class BaseConverter(object): IGNORE = "converter_ignore" @@ -6,6 +9,9 @@ def __init__(self): self.output_dir = None self.run_name = None self.input_assembly = None + self.module_name = self.__class__.__module__ + config_loader = ConfigLoader() + self.conf = config_loader.get_module_conf(self.module_name) def check_format(self, *args, **kwargs): err_msg = ( diff --git a/cravat/cravat_convert.py b/cravat/cravat_convert.py index a15df6ad..d4dc376a 100644 --- a/cravat/cravat_convert.py +++ b/cravat/cravat_convert.py @@ -260,6 +260,7 @@ def _initialize_converters(self): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) converter = module.CravatConverter() + converter.input_assembly = self.input_assembly if converter.format_name not in self.converters: self.converters[converter.format_name] = converter else: diff --git a/cravat/cravat_filter.py b/cravat/cravat_filter.py index e2c340dd..bb770483 100644 --- a/cravat/cravat_filter.py +++ b/cravat/cravat_filter.py @@ -564,10 +564,7 @@ async def getcount(self, level="variant", conn=None, cursor=None): ftable = level else: ftable = level + "_filtered" - q = "select count(*) from " + ftable - await cursor.execute(q) - for row in await cursor.fetchone(): - n = row + n = await self.exec_db(self.get_filtered_count, level=level) if self.stdout == True: print("#" + level) print(str(n)) @@ -645,8 +642,110 @@ async def getiterator(self, level="variant", conn=None, cursor=None): it = await cursor.fetchall() return it + @staticmethod + def reaggregate_column(base_alias, meta): + column = meta['name'] + function = meta.get('filter_reagg_function', None) + reagg_args = meta.get('filter_reagg_function_args', []) + reagg_source = meta.get('filter_reagg_source_column', None) + + if not function: + return "{}.{}".format(base_alias, column) + + reagg_template = "{}({}{}) OVER (PARTITION BY {}.base__uid ORDER BY sample.base__sample_id ROWS BETWEEN UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING) {}" + quoted_args = ["'{}'".format(x) for x in reagg_args] + formatted_args = ",{}".format(",".join(quoted_args)) if reagg_args else "" + return reagg_template.format(function, reagg_source, formatted_args, base_alias, column) + + @staticmethod + async def level_column_definitions(cursor, level): + await cursor.execute("select col_name, col_def from {}_header".format(level)) + return {k: json.loads(v) for k, v in await cursor.fetchall()} + + async def make_sample_filter_group(self, cursor, sample_filter): + sample_columns = await self.level_column_definitions(cursor, 'sample') + prefixes = {k: 'sample' for k in sample_columns.keys()} + filter_group = FilterGroup(sample_filter) + filter_group.add_prefixes(prefixes) + return filter_group + async def get_filtered_iterator(self, level="variant", conn=None, cursor=None): - bypassfilter = not(self.filter or self.filtersql or self.includesample or self.excludesample) + sql = await self.build_base_sql(cursor, level) + + if level == 'variant' and self.filter and 'samplefilter' in self.filter and len(self.filter['samplefilter']['rules']) > 0: + sample_filter = self.filter['samplefilter'] + variant_columns = await self.level_column_definitions(cursor, 'variant') + + reaggregated_columns = [self.reaggregate_column('v', meta) for col, meta in variant_columns.items()] + sample_filters = self.build_sample_exclusions() + filter_group = await self.make_sample_filter_group(cursor, sample_filter) + + sql = """ + with base_variant as ({}), + scoped_sample as ( + select * + from sample + where 1=1 + {} + ) + select distinct {} + from base_variant v + join scoped_sample sample on sample.base__uid = v.base__uid + where {} + """.format(sql, sample_filters, ",".join(reaggregated_columns), filter_group.get_sql()) + + await cursor.execute(sql) + cols = [v[0] for v in cursor.description] + rows = await cursor.fetchall() + + return cols, rows + + async def get_filtered_count(self, level="variant", conn=None, cursor=None): + + if level == 'variant' and self.filter and 'samplefilter' in self.filter and len(self.filter['samplefilter']['rules']) > 0: + sql = await self.build_base_sql(cursor, level) + sample_filter = self.filter['samplefilter'] + variant_columns = await self.level_column_definitions(cursor, 'variant') + + reaggregated_columns = [self.reaggregate_column('v', meta) for col, meta in variant_columns.items()] + sample_filters = self.build_sample_exclusions() + filter_group = await self.make_sample_filter_group(cursor, sample_filter) + + sql = """ + with base_variant as ({}), + scoped_sample as ( + select * + from sample + where 1=1 + {} + ) + select count(distinct v.base__uid) + from base_variant v + join scoped_sample sample on sample.base__uid = v.base__uid + where {} + """.format(sql, sample_filters, filter_group.get_sql()) + else: + sql = await self.build_base_sql(cursor, level, count=True) + await cursor.execute(sql) + rows = await cursor.fetchall() + + return rows[0][0] + + def build_sample_exclusions(self): + # this is needed because joining back to the sample table causes + # re-inclusion of sample data that was excluded at the variant level. + sample_filters = "" + req, rej = self.required_and_rejected_samples() + if req: + sample_filters += "and base__sample_id in ({})".format( + ", ".join(["'{}'".format(sid) for sid in req])) + if rej: + sample_filters += "and base__sample_id not in ({})".format( + ", ".join(["'{}'".format(sid) for sid in rej])) + return sample_filters + + async def build_base_sql(self, cursor, level, count=False): + bypassfilter = not (self.filter or self.filtersql or self.includesample or self.excludesample) if level == "variant": kcol = "base__uid" if bypassfilter: @@ -682,29 +781,20 @@ async def get_filtered_iterator(self, level="variant", conn=None, cursor=None): ", ".join(colnames) ) else: - sql = "select v.* from " + table + " as v" + if not count: + sql = "select v.* from " + table + " as v" + else: + sql = "select count(v.base__uid) from " + table + " as v" if bypassfilter == False: sql += " inner join " + ftable + " as f on v." + kcol + "=f." + kcol - await cursor.execute(sql) - cols = [v[0] for v in cursor.description] - rows = await cursor.fetchall() - return cols, rows + + return sql async def make_filtered_sample_table(self, conn=None, cursor=None): q = "drop table if exists fsample" await cursor.execute(q) await conn.commit() - req = [] - rej = [] - if "sample" in self.filter: - if "require" in self.filter["sample"]: - req = self.filter["sample"]["require"] - if "reject" in self.filter["sample"]: - rej = self.filter["sample"]["reject"] - if self.includesample is not None: - req = self.includesample - if self.excludesample is not None: - rej = self.excludesample + req, rej = self.required_and_rejected_samples() if len(req) > 0 or len(rej) > 0: q = "create table fsample as select distinct base__uid from sample" if req: @@ -721,6 +811,13 @@ async def make_filtered_sample_table(self, conn=None, cursor=None): else: return False + def required_and_rejected_samples(self): + sample = self.filter.get("sample", {}) + req = sample.get("require", self.includesample or []) + rej = sample.get("reject", self.excludesample or []) + + return req, rej + async def make_filter_where(self, conn=None, cursor=None): q = "" if len(self.filter) == 0: diff --git a/cravat/cravat_report.py b/cravat/cravat_report.py index 1900fd1b..ee64df73 100644 --- a/cravat/cravat_report.py +++ b/cravat/cravat_report.py @@ -613,6 +613,9 @@ async def get_variant_colinfo(self): if await self.exec_db(self.table_exists, level): await self.exec_db(self.make_col_info, level) level = "gene" + if await self.exec_db(self.table_exists, level): + await self.exec_db(self.make_col_info, level) + level = "sample" if await self.exec_db(self.table_exists, level): await self.exec_db(self.make_col_info, level) return self.colinfo diff --git a/cravat/cravat_web.py b/cravat/cravat_web.py index 14fb29d4..eb12fcf4 100644 --- a/cravat/cravat_web.py +++ b/cravat/cravat_web.py @@ -7,7 +7,6 @@ import json import sys import argparse -import imp import oyaml as yaml import re from cravat import admin_util as au diff --git a/cravat/inout.py b/cravat/inout.py index b17d033a..4908756b 100644 --- a/cravat/inout.py +++ b/cravat/inout.py @@ -543,6 +543,9 @@ def _load_dict(self, d): self.filterable = bool(d.get("filterable", True)) self.link_format = d.get("link_format") self.genesummary = d.get("genesummary", False) + self.filter_reagg_function = d.get("filter_reagg_function", None) + self.filter_reagg_function_args = d.get("filter_reagg_function_args", []) + self.filter_reagg_source_column = d.get("filter_reagg_source_column", None) self.table = d.get("table", False) def from_row(self, row, order=None): diff --git a/cravat/oc.py b/cravat/oc.py index 4cdfdda3..18244397 100644 --- a/cravat/oc.py +++ b/cravat/oc.py @@ -6,7 +6,7 @@ from cravat.cravat_report import parser as report_parser from cravat.vcfanno import vcfanno import sys -from pathlib import Path +import pathlib root_p = argparse.ArgumentParser( description="Open-CRAVAT genomic variant interpreter. https://github.com/KarchinLab/open-cravat" @@ -240,9 +240,16 @@ type = int, help = 'Number of CPU threads to use') vcfanno_p.add_argument('--temp-dir', - type = Path, - default = Path('temp-vcfanno'), + type = pathlib.Path, + default = pathlib.Path('temp-vcfanno'), help = 'Temporary directory for working files') +vcfanno_p.add_argument('-o','--output-path', + type = pathlib.Path, + help = 'Output vcf path (gzipped). Defaults to input_path.oc.vcf.gz') +vcfanno_p.add_argument('--chunk-size', + type = int, + default = 10**4, + help = 'Number of lines to annotate in each thread before syncing to disk. Affects performance.') vcfanno_p.set_defaults(func=vcfanno) def main(): diff --git a/cravat/vcfanno.py b/cravat/vcfanno.py index 3568985c..b1e58567 100644 --- a/cravat/vcfanno.py +++ b/cravat/vcfanno.py @@ -397,7 +397,10 @@ def process(self): def vcfanno(args): input_path = pathlib.Path(args.input_path) - output_path = pathlib.Path(str(input_path)+'.oc.vcf.gz') + if args.output_path is not None: + output_path = args.output_path + else: + output_path = pathlib.Path(str(input_path)+'.oc.vcf.gz') handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -413,7 +416,8 @@ def vcfanno(args): output_path = str(output_path), temp_dir = args.temp_dir, processors = args.threads if args.threads else mp.cpu_count(), - chunk_size=10**4, - chunk_log_frequency=50, - annotators=args.annotators) + chunk_size= args.chunk_size, + chunk_log_frequency = 50, + annotators = args.annotators, + ) anno.process() diff --git a/cravat/webresult/cravat_view.py b/cravat/webresult/cravat_view.py deleted file mode 100644 index b9412460..00000000 --- a/cravat/webresult/cravat_view.py +++ /dev/null @@ -1,505 +0,0 @@ -from http.server import HTTPServer, CGIHTTPRequestHandler -from socketserver import TCPServer -import os -import webbrowser -import multiprocessing -import sqlite3 -import urllib.parse -import json -import sys -import argparse -import imp -import yaml -import re -from cravat import ConfigLoader -from cravat import admin_util as au -from cravat import CravatFilter - -def get (handler): - head = handler.trim_path_head() - if head == 'service': - serve_service(handler) - elif head == 'widgetfile': - serve_widgetfile(handler) - elif head == 'runwidget': - serve_runwidget(handler) - else: - handler.request_path = head + '/' + handler.request_path - handler.request_path = handler.request_path.rstrip('/') - filepath = get_filepath(handler.request_path) - serve_view(handler, filepath) - -### files ### - -def get_filepath (path): - filepath = os.sep.join(path.split('/')) - filepath = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - 'webviewer', - filepath - ) - return filepath - -def serve_view (handler, filepath): - handler.send_response(200) - if filepath[-4:] == '.css': - handler.send_header('Content-type', 'text/css') - elif filepath[-3:] == '.js': - handler.send_header('Content-type', 'application/javascript') - elif filepath[-4:] == '.png': - handler.send_header('Content-type', 'image/png') - elif filepath[-4:] == '.jpg': - handler.send_header('Content-type', 'image/jpg') - elif filepath[-4:] == '.gif': - handler.send_header('Content-type', 'image/gif') - else: - handler.send_header('Content-type', 'text/html') - handler.end_headers() - with open(filepath, 'rb') as f: - response = f.read() - handler.wfile.write(response) - -### service ### - -def serve_service (handler): - head = handler.trim_path_head() - queries = handler.request_queries - handler.send_response(200) - handler.send_header('Content-type', 'application/json') - handler.end_headers() - if head == 'variantcols': - content = get_variant_cols(queries) - #if head == 'conf': - # content = get_webviewerconf() - elif head == 'getsummarywidgetnames': - content = get_summary_widget_names(queries) - elif head == 'getresulttablelevels': - content = get_result_levels(queries) - elif head == 'result': - content = get_result(queries) - elif head == 'count': - content = get_count(queries) - elif head == 'widgetlist': - content = get_widgetlist() - elif head == 'status': - content = get_status(queries) - elif head == 'savefiltersetting': - content = save_filter_setting(queries) - elif head == 'savelayoutsetting': - content = save_layout_setting(queries) - elif head == 'loadfiltersetting': - content = load_filtersetting(queries) - elif head == 'loadlayoutsetting': - content = load_layout_setting(queries) - elif head == 'deletelayoutsetting': - content = delete_layout_setting(queries) - elif head == 'renamelayoutsetting': - content = rename_layout_setting(queries) - elif head == 'getlayoutsavenames': - content = get_layout_save_names(queries) - elif head == 'getfiltersavenames': - content = get_filter_save_names(queries) - elif head == 'getnowgannotmodules': - content = get_nowg_annot_modules(queries) - handler.response = bytes(json.dumps(content), 'UTF-8') - handler.wfile.write(handler.response) - -def get_nowg_annot_modules (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - wgmodules = au.get_local_module_infos_of_type('webviewerwidget') - annot_modules_with_wg = [] - for wgmodule in wgmodules: - conf = wgmodules[wgmodule].conf - if 'required_annotator' in conf: - if wgmodule not in annot_modules_with_wg: - annot_modules_with_wg.append(wgmodule) - nowg_annot_modules = {} - if table_exists(cursor, 'variant'): - q = 'select name, displayname from variant_annotator' - cursor.execute(q) - for r in cursor.fetchall(): - m = r[0] - if m in ['example_annotator', 'testannot', 'tagsampler']: - continue - annot_module = 'wg' + r[0] - displayname = r[1] - if annot_module not in annot_modules_with_wg and annot_module not in nowg_annot_modules: - nowg_annot_modules[annot_module] = displayname - content = nowg_annot_modules - return content - -def get_filter_save_names (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - table = 'viewersetup' - if table_exists(cursor, table) == False: - content = [] - else: - q = 'select distinct name from ' + table + ' where datatype="filter"' - cursor.execute(q) - r = cursor.fetchall() - content = str([v[0] for v in r]) - cursor.close() - conn.close() - return content - -def get_layout_save_names (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - table = 'viewersetup' - content = [] - if table_exists(cursor, table): - q = 'select distinct name from ' + table + ' where datatype="layout"' - cursor.execute(q) - r = cursor.fetchall() - content = [v[0] for v in r] - cursor.close() - conn.close() - return content - -def rename_layout_setting (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - name = urllib.parse.unquote(queries['name'][0]) - new_name = urllib.parse.unquote(queries['newname'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - table = 'viewersetup' - if table_exists(cursor, table) == True: - q = 'update ' + table + ' set name="' + new_name + '" where datatype="layout" and name="' + name + '"' - cursor.execute(q) - conn.commit() - cursor.close() - conn.close() - content = {} - return content - -def delete_layout_setting (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - name = urllib.parse.unquote(queries['name'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - table = 'viewersetup' - if table_exists(cursor, table) == True: - q = 'DELETE FROM ' + table + ' WHERE datatype="layout" and name="' + name + '"' - cursor.execute(q) - conn.commit() - cursor.close() - conn.close() - content = {} - return content - -def load_layout_setting (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - name = urllib.parse.unquote(queries['name'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - table = 'viewersetup' - if table_exists(cursor, table) == False: - content = {"widgetSettings": []} - else: - q = 'select viewersetup from ' + table + ' where datatype="layout" and name="' + name + '"' - cursor.execute(q) - r = cursor.fetchone() - if r != None: - data = r[0] - content = json.loads(data) - else: - content = {"widgetSettings": []} - cursor.close() - conn.close() - return content - -def load_filtersetting (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - name = urllib.parse.unquote(queries['name'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - table = 'viewersetup' - if table_exists(cursor, table) == False: - content = {"filterSet": []} - else: - q = 'select viewersetup from ' + table + ' where datatype="filter" and name="' + name + '"' - cursor.execute(q) - r = cursor.fetchone() - if r != None: - data = r[0] - content = json.loads(data) - else: - content = {"filterSet": []} - cursor.close() - conn.close() - return content - -def save_layout_setting (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - name = urllib.parse.unquote(queries['name'][0]) - savedata = urllib.parse.unquote(queries['savedata'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - table = 'viewersetup' - if table_exists(cursor, table) == False: - q = 'create table ' + table + ' (datatype text, name text, viewersetup text, unique (datatype, name))' - cursor.execute(q) - q = 'replace into ' + table + ' values ("layout", "' + name + '", \'' + savedata + '\')' - cursor.execute(q) - conn.commit() - cursor.close() - conn.close() - content = 'saved' - return content - -def save_filter_setting (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - name = urllib.parse.unquote(queries['name'][0]) - savedata = urllib.parse.unquote(queries['savedata'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - table = 'viewersetup' - if table_exists(cursor, table) == False: - q = 'create table ' + table + ' (datatype text, name text, viewersetup text, unique (datatype, name))' - cursor.execute(q) - q = 'replace into ' + table + ' values ("filter", "' + name + '", \'' + savedata + '\')' - cursor.execute(q) - conn.commit() - cursor.close() - conn.close() - content = 'saved' - return content - -def get_status (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - conn = sqlite3.connect(dbpath) - cursor = conn.cursor() - q = 'select * from info' - cursor.execute(q) - content = {} - for row in cursor.fetchall(): - content[row[0]] = row[1] - return content - -def get_widgetlist (): - content = [] - modules = au.get_local_module_infos_of_type('webviewerwidget') - for module_name in modules: - module = modules[module_name] - conf = module.conf - if 'required_annotator' in conf: - req = conf['required_annotator'] - else: - # Removes wg. - req = module_name[2:] - content.append({'name': module_name, - 'title': module.title, - 'required_annotator': req}) - return content - -def get_count (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - tab = queries['tab'][0] - if 'filter' in queries: - filterstring = queries['filter'][0] - else: - filterstring = None - cf = CravatFilter(dbpath=dbpath, - mode='sub', - filterstring=filterstring) - n = cf.exec_db(cf.getcount, level=tab) - content = {'n': n} - return content - -def get_result (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - tab = queries['tab'][0] - if 'filter' in queries: - filterstring = queries['filter'][0] - else: - filterstring = None - if 'confpath' in queries: - confpath = queries['confpath'][0] - else: - confpath = None - reporter_name = 'jsonreporter' - f, fn, d = imp.find_module( - reporter_name, - [os.path.join(os.path.dirname(__file__), 'webviewer')]) - m = imp.load_module(reporter_name, f, fn, d) - args = ['', dbpath] - if confpath != None: - args.extend(['-c', confpath]) - if filterstring != None: - args.extend(['--filterstring', filterstring]) - reporter = m.Reporter(args) - data = reporter.run(tab=tab) - content = {} - content['stat'] = {'rowsreturned': True, - 'wherestr':'', - 'filtered': True, - 'filteredresultmessage': '', - 'maxnorows': 100000, - 'norows': data['info']['norows']} - content['columns'] = get_colmodel(tab, data['colinfo']) - content["data"] = get_datamodel(data[tab]) - content["status"] = "normal" - return content - -def get_result_levels (queries): - conn = sqlite3.connect(queries['dbpath'][0]) - cursor = conn.cursor() - sql = 'select name from sqlite_master where type="table" and ' +\ - 'name like "%_header"' - cursor.execute(sql) - ret = cursor.fetchall() - if len(ret) > 0: - content = [v[0].split('_')[0] for v in ret] - content.insert(0, 'info') - else: - content = [] - return content - -def get_variant_cols (queries): - dbpath = urllib.parse.unquote(queries['dbpath'][0]) - if 'confpath' in queries: - confpath = queries['confpath'][0] - else: - confpath = None - if 'filter' in queries: - filterstring = queries['filter'][0] - else: - filterstring = None - data = {} - data['data'] = {} - data['stat'] = {} - data['status'] = {} - colinfo = get_colinfo(dbpath, confpath, filterstring) - data['columns'] = {} - if 'variant' in colinfo: - data['columns']['variant'] = get_colmodel('variant', colinfo) - if 'gene' in colinfo: - data['columns']['gene'] = get_colmodel('gene', colinfo) - content = data - return content - -def get_webviewerconf (): - conf_path = os.path.join( - au.get_system_conf()['home'], - 'viewers', - 'webviewer', - 'webviewer.yml') - with open(conf_path) as f: - conf = yaml.safe_load(f) - return conf - -def get_summary_widget_names (queries): - runid = queries['jobid'][0] - -def get_datamodel (data): - ret = [] - for row in data: - ret.append(list(row)) - return ret - -def get_colmodel (tab, colinfo): - colModel = [] - groupkeys_ordered = [] - groupnames = {} - for d in colinfo[tab]['colgroups']: - groupnames[d['name']] = [d['displayname'], d['count']] - groupkeys_ordered.append(d['name']) - dataindx = 0 - for groupkey in groupkeys_ordered: - [grouptitle, col_count] = groupnames[groupkey] - columngroupdef = {'title': grouptitle, 'colModel': []} - startidx = dataindx - endidx = startidx + col_count - for d in colinfo[tab]['columns'][startidx:endidx]: - column = { - "col": d['col_name'], - 'colgroupkey': groupkey, - 'colgroup': grouptitle, - "title": d['col_title'], - "align":"center", - "hidden":False, - "dataIndx": dataindx, - "retfilt":False, - "retfilttype":"None", - "multiseloptions":[] - } - if d['col_type'] == 'string': - column['filter'] = { - "type":"textbox", - "condition":"contain", - "listeners":["keyup"]} - column['retfilt'] = True - column['retfilttype'] = 'regexp' - column['multiseloptions'] = [] - elif d['col_type'] == 'float' or d['col_type'] == 'int': - column['filter'] = { - "type":"textbox", - "condition":"between", - "listeners":["keyup"]} - column['retfilt'] = True - column['retfilttype'] = 'between' - column['multiseloptions'] = [] - columngroupdef['colModel'].append(column) - dataindx += 1 - colModel.append(columngroupdef) - return colModel - -def get_colinfo (dbpath, confpath, filterstring): - reporter_name = 'jsonreporter' - f, fn, d = imp.find_module( - reporter_name, - [os.path.join(os.path.dirname(__file__), 'webviewer')]) - m = imp.load_module(reporter_name, f, fn, d) - args = ['', dbpath] - if confpath != None: - args.extend(['-c', confpath]) - if filterstring != None: - args.extend(['--filterstring', filterstring]) - reporter = m.Reporter(args) - colinfo = reporter.get_variant_colinfo() - return colinfo - -def table_exists (cursor, table): - sql = 'select name from sqlite_master where type="table" and ' +\ - 'name="' + table + '"' - cursor.execute(sql) - if cursor.fetchone() == None: - return False - else: - return True - -### widgetfiles ### - -def serve_widgetfile (handler): - module_name, ext = os.path.splitext(handler.request_path) - filepath = os.path.join( - au.get_modules_dir(), - 'webviewerwidgets', - handler.request_path) - #module_name, - #module_name + ext) - if os.path.exists(filepath): - serve_view(handler, filepath) - -### runwidget ### - -def serve_runwidget (handler): - path = 'wg' + handler.request_path - queries = handler.request_queries - f, fn, d = imp.find_module(path, - [os.path.join(au.get_modules_dir(), - 'webviewerwidgets', path)]) - m = imp.load_module(path, f, fn, d) - ret = m.get_data(queries) - handler.send_response(200) - handler.send_header('Content-type', 'application/json') - handler.end_headers() - content = json.dumps(ret) - response = bytes(content, 'UTF-8') - handler.wfile.write(response) - diff --git a/cravat/webresult/nocache/js/filter.js b/cravat/webresult/nocache/js/filter.js index 89cdfd6a..4d912d22 100644 --- a/cravat/webresult/nocache/js/filter.js +++ b/cravat/webresult/nocache/js/filter.js @@ -98,7 +98,7 @@ const makeDragHandle = function () { return moveDiv } -const makeFilterColDiv = (filter) => { +const makeFilterColDiv = (filter, sampfilt) => { const colDiv = $(getEl('div')) .addClass('filter-column-div') .addClass('filter-element-div'); @@ -111,10 +111,17 @@ const makeFilterColDiv = (filter) => { colDiv.append(groupSel); for (let i=0; i { +const makeFilterGroupDiv = (filter, sampfilt) => { // Group div const groupDiv = $(getEl('div')) .addClass('filter-group-div') @@ -459,7 +466,7 @@ const makeFilterGroupDiv = (filter) => { .addClass('addrule') .addClass('hovercontrol') .click(function (evt) { - addFilterRuleHandler(evt); + addFilterRuleHandler(evt, sampfilt); }) .attr('title','Add rule'); groupDiv.append(addRuleBtn); @@ -470,7 +477,7 @@ const makeFilterGroupDiv = (filter) => { .addClass('addgroup') .addClass('hovercontrol') .click(function (evt) { - addFilterGroupHandler(evt); + addFilterGroupHandler(evt), sampfilt; }) .attr('title','Add group'); groupDiv.append(addGroupBtn); @@ -487,9 +494,9 @@ const makeFilterGroupDiv = (filter) => { for (let i=0; i { } } } else { - addFilterElement($(elemsDiv),'rule', undefined); + addFilterElement($(elemsDiv),'rule', undefined, sampfilt); } return groupDiv; } @@ -525,16 +532,16 @@ const removeFilterElem = (elemDiv) => { elemDiv.remove(); } -const addFilterRuleHandler = (event) => { +const addFilterRuleHandler = (event, sampfilt) => { const button = event.target const elemsDiv = $(button.closest('.filter-group-div').querySelector('div.filter-group-elements-div')) - addFilterElement(elemsDiv, 'rule', undefined); + addFilterElement(elemsDiv, 'rule', undefined, sampfilt); } -const addFilterGroupHandler = (event) => { +const addFilterGroupHandler = (event, sampfilt) => { const button = event.target const elemsDiv = $(button.closest('.filter-group-div').querySelector('div.filter-group-elements-div')) - addFilterElement(elemsDiv, 'group', undefined); + addFilterElement(elemsDiv, 'group', undefined, sampfilt); } const makeJoinOperatorDiv = function (operator) { @@ -580,12 +587,12 @@ const makeJoinOperatorDiv = function (operator) { return joinOpDiv } -const addFilterElement = (allElemsDiv, elementType, filter) => { +const addFilterElement = (allElemsDiv, elementType, filter, sampfilt) => { let elemDiv; if (elementType === 'group') { - elemDiv = makeFilterGroupDiv(filter); + elemDiv = makeFilterGroupDiv(filter, sampfilt); } else if (elementType === 'rule') { - elemDiv = makeFilterColDiv(filter); + elemDiv = makeFilterColDiv(filter, sampfilt); } elemDiv[0].id = 'filter-element-' + filterElemCount filterElemCount += 1 diff --git a/cravat/webresult/nocache/js/main.js b/cravat/webresult/nocache/js/main.js index 47daaf75..19a73d33 100644 --- a/cravat/webresult/nocache/js/main.js +++ b/cravat/webresult/nocache/js/main.js @@ -1091,24 +1091,21 @@ function afterGetResultLevels () { }); jobDataLoadingDiv = drawingRetrievingDataDiv(currentTab); $.get('/result/service/variantcols', {job_id: jobId, username: username, dbpath: dbPath, confpath: confPath, filter: JSON.stringify(filterJson)}).done(function (jsonResponseData) { - allVarCols = JSON.parse(JSON.stringify(jsonResponseData['columns']['variant'])); - filterCols = []; - for (let colGroup of allVarCols) { + filterCols = JSON.parse(JSON.stringify(jsonResponseData['columns']['variant'])); + let vcfInfoTypes = {} + for (let colGroup of jsonResponseData['columns']['sample']) { + for (let col of colGroup.colModel) { + vcfInfoTypes[col.col] = col.type; + } + } + for (let colGroup of filterCols) { if (colGroup.name === 'vcfinfo') { - let filterableCols = []; for (let col of colGroup.colModel) { - if (col.filterable) { - filterableCols.push(col); - } + colNameToks = col.col.split('__'); + baseColName = `base__${colNameToks[1]}` + col.type = vcfInfoTypes[baseColName]; } - if (filterableCols.length > 0) { - filterCols.push(colGroup); - filterCols.slice(-1)[0].colModel = filterableCols; - } - } else { - filterCols.push(colGroup) } - } usedAnnotators = {}; var cols = jsonResponseData['columns']['variant']; diff --git a/cravat/webresult/nocache/js/setup.js b/cravat/webresult/nocache/js/setup.js index bf9afb5d..08aedb4e 100644 --- a/cravat/webresult/nocache/js/setup.js +++ b/cravat/webresult/nocache/js/setup.js @@ -82,7 +82,7 @@ class FilterManager { this.sampleContId = 'filter-cont-sample'; this.geneContId = 'filter-cont-gene'; this.variantContId = 'filter-cont-variant' - + this.sampFiltContId = 'filter-cont-sampfilt'; this.sampleControlClass = 'sample-control' this.sampleFilterId = 'sample-select-filt'; this.sampleFileId = 'sample-list-file'; @@ -100,7 +100,8 @@ class FilterManager { this.vpropSelectId = 'vprop-sel'; this.vpropSfId = 'vprop-sf'; this.vpropQbId = 'vprop-qb'; - this.qbRootId = 'qb-root'; + this.qbVarRootId = 'qb-root'; + this.qbSampRootId = 'qb-samp-root'; this.qbBannedColumns = [ //'base__numsample', 'base__samples', @@ -527,6 +528,56 @@ class FilterManager { } } + addSPropUI (sPropCont, filter) { + filter = new CravatFilter(filter); + sPropCont.attr('id', this.sampFiltContId); + // sPropCont.append($(getEl('div')) + // .text('Select variants by applying filters or building a query')) + // let fTypeDiv = $(getEl('div')); + // sPropCont.append(fTypeDiv); + // let vPropSel = $(getEl('select')) + // .attr('id', this.vpropSelectId) + // .append($(getEl('option')).val('sf').text('sf')) + // .append($(getEl('option')).val('qb').text('qb')) + // .css('display','none') + // .change(this.vPropSelectChange.bind(this)); + // fTypeDiv.append(vPropSel); + // let sfHeader = $(getEl('span')) + // .addClass('vprop-option') + // .text('Smart Filters') + // .addClass('title') + // .click(this.vPropOptionClick) + // .attr('value','sf'); + // fTypeDiv.append(sfHeader); + // let qbHeader = $(getEl('span')) + // .addClass('vprop-option') + // .text('Query Builder') + // .addClass('title') + // .click(this.vPropOptionClick) + // .attr('value','qb'); + // fTypeDiv.append(qbHeader); + // let sfContent = $(getEl('div')) + // .attr('id', this.vpropSfId); + // sPropCont.append(sfContent); + // this.addSfUI(sfContent, filter); + // let qbContent = $(getEl('div')) + // .attr('id', this.vpropQbId); + // sPropCont.append(qbContent); + this.addQbUI(sPropCont, filter, 'sample'); + + // Activate the correct vProp type + // let sfValued = Object.keys(filter.smartfilter).length !== 0; + // let qbValued = filter.variant.rules!==undefined && filter.variant.rules.length>0; + // if (sfValued || !qbValued) { + // vPropSel.val('sf'); + // sfHeader.addClass('active'); + // } else { + // vPropSel.val('qb'); + // qbHeader.addClass('active'); + // } + // vPropSel.change(); + } + addVpropUI (vPropCont, filter) { filter = new CravatFilter(filter); vPropCont.attr('id', this.variantContId); @@ -607,17 +658,22 @@ class FilterManager { } } - addQbUI (outerDiv, filter) { + addQbUI (outerDiv, filter, QbType) { filter = new CravatFilter(filter); - outerDiv.append($(getEl('div')).text('Use the query builder to create a set of filter rules')); - let qbDiv = makeFilterGroupDiv(filter.variant); + if (QbType === 'sample') { + outerDiv.append($(getEl('div')).text('Variants are returned if any sample matches the criteria')); + var qbDiv = makeFilterGroupDiv(filter.samplefilter, true); + } else { + outerDiv.append($(getEl('div')).text('Use the query builder to create a set of filter rules')); + var qbDiv = makeFilterGroupDiv(filter.variant); + } qbDiv.children('.filter-element-control-div').remove() qbDiv[0].querySelector('.addrule').style.visibility = 'visible' qbDiv[0].querySelector('.addgroup').style.visibility = 'visible' qbDiv[0].querySelector('.passthrough').remove() - qbDiv.attr('id', this.qbRootId); + let divId = QbType==='sample' ? this.qbSampRootId : this.qbVarRootId + qbDiv.attr('id', divId); outerDiv.append(qbDiv); - //addFilterElement(qbDiv, 'rule', undefined) } updateVpropUI (filter) { @@ -671,6 +727,7 @@ class CravatFilter { this.smartfilter = f.smartfilter!==undefined ? f.smartfilter : {}; this.genes = f.genes!==undefined ? f.genes : []; this.sample = f.sample!==undefined ? f.sample : {require:[],reject:[]}; + this.samplefilter = f.samplefilter!==undefined ? f.samplefilter : {operator:'and',rules:[]} } } @@ -948,7 +1005,13 @@ function makeFilterTab (rightDiv) { let geneContent = geneSection.find('.filter-content'); filterMgr.addGeneSelect(geneContent); - // Smartfilters + // Sample properties + let sPropSection = filterMgr.getFilterSection('Sample Properties',false); + rightPanel.append(sPropSection); + let sPropContent = sPropSection.find('.filter-content'); + filterMgr.addSPropUI(sPropContent); + + // Smartfilters let vPropSection = filterMgr.getFilterSection('Variant Properties', true); rightPanel.append(vPropSection); let vPropContent = vPropSection.find('.filter-content'); @@ -1111,7 +1174,7 @@ function makeFilterJson () { .map(s=>s.toUpperCase()) .filter(s=>s); fjs.genes = geneList; - // Variant Properties + // Variant properties let activeVprop = $('#'+filterMgr.vpropSelectId).val(); if (activeVprop === 'sf') { let sfWrapDiv = $('#'+filterMgr.vpropSfId); @@ -1138,10 +1201,18 @@ function makeFilterJson () { fjs.variant = fullSf; fjs.smartfilter = sfState; } else if (activeVprop === 'qb') { - let qbRoot = $('#'+filterMgr.qbRootId); + let qbRoot = $('#'+filterMgr.qbVarRootId); fjs.variant = makeGroupFilter(qbRoot); fjs.smartfilter = {}; } + // Sample filter + let spropRoot = $('#'+filterMgr.qbSampRootId); + fjs.samplefilter = makeGroupFilter(spropRoot); + for (let rule of fjs.samplefilter.rules) { + rule.column = rule.column.replace('vcfinfo__','base__') + } + + // Set global filterJson = fjs; } diff --git a/cravat/webresult/webresult.py b/cravat/webresult/webresult.py index e4dba579..c4efd352 100644 --- a/cravat/webresult/webresult.py +++ b/cravat/webresult/webresult.py @@ -3,7 +3,8 @@ import sqlite3 import json import sys -import imp +import importlib +from importlib import util as importlib_util from cravat import admin_util as au from cravat import CravatFilter from cravat.constants import base_smartfilters @@ -330,10 +331,10 @@ async def get_result (request): else: confpath = None reporter_name = 'jsonreporter' - f, fn, d = imp.find_module( - reporter_name, - [os.path.join(os.path.dirname(__file__),)]) - m = imp.load_module(reporter_name, f, fn, d) + fp = os.path.join(os.path.dirname(__file__), f'{reporter_name}.py') + spec = importlib_util.spec_from_file_location(reporter_name, fp) + m = importlib_util.module_from_spec(spec) + spec.loader.exec_module(m) arg_dict = {'dbpath': dbpath, 'module_name': reporter_name} if confpath != None: arg_dict['confpath'] = confpath @@ -445,6 +446,8 @@ async def get_variant_cols (request): data['columns']['variant'] = get_colmodel('variant', colinfo) if 'gene' in colinfo: data['columns']['gene'] = get_colmodel('gene', colinfo) + if 'sample' in colinfo: + data['columns']['sample'] = get_colmodel('sample', colinfo) content = data return web.json_response(content) @@ -546,10 +549,10 @@ def get_colmodel (tab, colinfo): async def get_colinfo (dbpath, confpath, filterstring): reporter_name = 'jsonreporter' - f, fn, d = imp.find_module( - reporter_name, - [os.path.join(os.path.dirname(__file__),)]) - m = imp.load_module(reporter_name, f, fn, d) + fp = os.path.join(os.path.dirname(__file__), f'{reporter_name}.py') + spec = importlib_util.spec_from_file_location(reporter_name, fp) + m = importlib_util.module_from_spec(spec) + spec.loader.exec_module(m) arg_dict = {'dbpath': dbpath, 'module_name': reporter_name} if confpath != None: arg_dict['confpath'] = confpath @@ -603,10 +606,10 @@ async def serve_runwidget (request): if key != 'dbpath': new_queries[key] = queries[key] queries = new_queries - f, fn, d = imp.find_module(path, - [os.path.join(au.get_modules_dir(), - 'webviewerwidgets', path)]) - m = imp.load_module(path, f, fn, d) + info = au.get_local_module_info(path) + spec = importlib_util.spec_from_file_location(path, info.script_path) + m = importlib_util.module_from_spec(spec) + spec.loader.exec_module(m) content = await m.get_data(queries) return web.json_response(content) @@ -618,15 +621,16 @@ async def serve_webapp_runwidget (request): for key in queries: tmp_queries[key] = queries[key] queries = tmp_queries - f, fn, d = imp.find_module( - 'wg' + widget_name, - [os.path.join(au.get_modules_dir(), 'webapps', module_name, 'widgets', 'wg' + widget_name)] - ) - m = imp.load_module(widget_name, f, fn, d) + name = f'wg{widget_name}' + widget_path = os.path.join(au.get_modules_dir(), 'webapps', module_name, 'widgets', name, f'{name}.py') + spec = importlib_util.spec_from_file_location(name, widget_path) + m = importlib_util.module_from_spec(spec) + spec.loader.exec_module(m) content = await m.get_data(queries) return web.json_response(content) async def serve_runwidget_post (request): + # NOTE: This method seems to not be called either in the OpenCravat or OpenCravat modules code path = 'wg' + request.match_info['module'] job_id, dbpath = await get_jobid_dbpath(request) queries = await request.post() @@ -653,10 +657,10 @@ async def serve_runwidget_post (request): if key != 'dbpath': new_queries[key] = queries[key] queries = new_queries - f, fn, d = imp.find_module(path, - [os.path.join(au.get_modules_dir(), - 'webviewerwidgets', path)]) - m = imp.load_module(path, f, fn, d) + info = au.get_local_module_info(path) + spec = importlib_util.spec_from_file_location(path, info.script_path) + m = importlib_util.module_from_spec(spec) + spec.loader.exec_module(m) content = await m.get_data(queries) return web.json_response(content)