From 75da9156054903e4975dc14ce6c391493ed2d68f Mon Sep 17 00:00:00 2001 From: Sigfried Gold Date: Wed, 17 Jan 2024 14:51:40 -0500 Subject: [PATCH] This should finish #673 - Added total_cnt_from_term_usage to all_csets - Commented out unneeded parameters in _get_cset_members_items - Improved get_comparison_rpt but left old diff stuff so it doesn't break prod/dev until new front end code is deployed. - Added std concept as column to main comparison table and to N3C comparison report diff list --- backend/db/ddl-11-all_csets.jinja.sql | 17 ++++- backend/routes/db.py | 58 ++++++++------ frontend/src/App.css | 22 ++++++ frontend/src/components/ConceptSetCard.jsx | 2 + .../src/components/CsetComparisonPage.jsx | 7 ++ frontend/src/components/N3CRecommended.jsx | 76 +++++++++---------- 6 files changed, 117 insertions(+), 65 deletions(-) diff --git a/backend/db/ddl-11-all_csets.jinja.sql b/backend/db/ddl-11-all_csets.jinja.sql index 9b698cca1..2a96d6376 100644 --- a/backend/db/ddl-11-all_csets.jinja.sql +++ b/backend/db/ddl-11-all_csets.jinja.sql @@ -1,6 +1,15 @@ -- Table: all_csets ---------------------------------------------------------------------------------------------------- DROP TABLE IF EXISTS {{schema}}all_csets{{optional_suffix}} CASCADE; +CREATE TABLE {{schema}}cset_term_usage_rec_counts{{optional_suffix}} AS + SELECT csm.codeset_id, SUM(cwc.total_cnt) AS total_cnt + FROM {{schema}}concept_set_members csm + JOIN {{schema}}concepts_with_counts cwc ON csm.concept_id = cwc.concept_id + WHERE cwc.total_cnt > 0 + GROUP BY csm.codeset_id; + +CREATE INDEX ctu_idx1{{optional_index_suffix}} ON {{schema}}cset_term_usage_rec_counts{{optional_suffix}}(codeset_id); + CREATE TABLE {{schema}}all_csets{{optional_suffix}} AS -- table instead of view for performance (no materialized views in mySQL) -- TODO: but now we're on postgres should it be a materialized view? @@ -45,13 +54,15 @@ WITH ac AS (SELECT DISTINCT cs.codeset_id, -- COALESCE(members.concepts, 0) AS members, -- COALESCE(items.concepts, 0) AS items, COALESCE(cscc.approx_distinct_person_count, 0) AS distinct_person_cnt, - COALESCE(cscc.approx_total_record_count, 0) AS total_cnt + COALESCE(cscc.approx_total_record_count, 0) AS total_cnt, + COALESCE(ctu.total_cnt, 0) AS total_cnt_from_term_usage FROM {{schema}}code_sets cs LEFT JOIN {{schema}}OMOPConceptSet ocs ON cs.codeset_id = ocs."codesetId" -- need quotes because of caps in colname JOIN {{schema}}concept_set_container csc ON cs.concept_set_name = csc.concept_set_name LEFT JOIN {{schema}}omopconceptsetcontainer ocsc ON csc.concept_set_id = ocsc."conceptSetId" LEFT JOIN {{schema}}concept_set_counts_clamped cscc ON cs.codeset_id = cscc.codeset_id + LEFT JOIN {{schema}}cset_term_usage_rec_counts ctu ON cs.codeset_id = ctu.codeset_id /* want to add term usage record counts, tried this code: COALESCE(cwc.total_cnt, 0) AS total_cnt_from_term_usage @@ -75,4 +86,6 @@ LEFT JOIN {{schema}}researcher rver ON ac.codeset_created_by = rver."multipassId CREATE INDEX ac_idx1{{optional_index_suffix}} ON {{schema}}all_csets{{optional_suffix}}(codeset_id); -CREATE INDEX ac_idx2{{optional_index_suffix}} ON {{schema}}all_csets{{optional_suffix}}(concept_set_name); \ No newline at end of file +CREATE INDEX ac_idx2{{optional_index_suffix}} ON {{schema}}all_csets{{optional_suffix}}(concept_set_name); + +DROP TABLE {{schema}}cset_term_usage_rec_counts{{optional_suffix}}; \ No newline at end of file diff --git a/backend/routes/db.py b/backend/routes/db.py index 7718e74c5..a70ee295e 100644 --- a/backend/routes/db.py +++ b/backend/routes/db.py @@ -13,7 +13,7 @@ from starlette.responses import Response from backend.api_logger import Api_logger -from backend.utils import get_timer, return_err_with_trace +from backend.utils import get_timer, return_err_with_trace, commify from backend.db.utils import get_db_connection, sql_query, SCHEMA, sql_query_single_col, sql_in, run_sql from backend.db.queries import get_concepts from enclave_wrangler.objects_api import get_n3c_recommended_csets, enclave_api_call_caller, \ @@ -51,19 +51,17 @@ # probably don't need precision etc. # switched _container suffix on duplicate col names to container_ prefix # joined OMOPConceptSet in the all_csets ddl to get `rid` -def get_csets(codeset_ids: List[int], con: Connection = None) -> List[Dict]: +def get_csets(codeset_ids: List[int]) -> List[Dict]: """Get information about concept sets the user has selected""" - conn = con if con else get_db_connection() - rows: List = sql_query( - conn, """ - SELECT * - FROM all_csets - WHERE codeset_id = ANY(:codeset_ids);""", - {'codeset_ids': codeset_ids}) + with get_db_connection() as con: + rows: List = sql_query( + con, """ + SELECT * + FROM all_csets + WHERE codeset_id = ANY(:codeset_ids);""", + {'codeset_ids': codeset_ids}) # {'codeset_ids': ','.join([str(id) for id in requested_codeset_ids])}) row_dicts = [dict(x) for x in rows] - if not con: - conn.close() for row in row_dicts: row['researchers'] = get_row_researcher_ids_dict(row) @@ -137,8 +135,8 @@ def _cset_members_items(codeset_ids: Union[str, None] = Query(default=''), ) -> @router.get("/get-cset-members-items") async def _get_cset_members_items(request: Request, codeset_ids: str, - columns: Union[List[str], None] = Query(default=None), - column: Union[str, None] = Query(default=None), + # columns: Union[List[str], None] = Query(default=None), + # column: Union[str, None] = Query(default=None), # extra_concept_ids: Union[int, None] = Query(default=None) ) -> Union[List[int], List]: requested_codeset_ids = parse_codeset_ids(codeset_ids) @@ -146,7 +144,7 @@ async def _get_cset_members_items(request: Request, await rpt.start_rpt(request, params={'codeset_ids': requested_codeset_ids}) try: - rows = get_cset_members_items(requested_codeset_ids, columns, column) + rows = get_cset_members_items(requested_codeset_ids) #, columns, column await rpt.finish(rows=len(rows)) except Exception as e: await rpt.log_error(e) @@ -606,7 +604,6 @@ def get_comparison_rpt(con, codeset_id_1: int, codeset_id_2: int) -> Dict[str, U SELECT concept_id, concept_name FROM concept_set_members WHERE codeset_id = :cset_2_codeset_id ) x """, {'codeset_id_1': codeset_id_1, 'cset_2_codeset_id': codeset_id_2}) - # orig_only = [dict(r) for r in orig_only] cset_1_only = [dict(r)['diff'] for r in cset_1_only] cset_2_only = sql_query(con, """ @@ -616,11 +613,22 @@ def get_comparison_rpt(con, codeset_id_1: int, codeset_id_2: int) -> Dict[str, U SELECT concept_id, concept_name FROM concept_set_members WHERE codeset_id = :codeset_id_1 ) x """, {'codeset_id_1': codeset_id_1, 'cset_2_codeset_id': codeset_id_2}) - # cset_2_only = [dict(r) for r in cset_2_only] cset_2_only = [dict(r)['diff'] for r in cset_2_only] diffs = cset_1_only + cset_2_only + removed = sql_query_single_col(con, """ + SELECT concept_id FROM concept_set_members WHERE codeset_id = :codeset_id_1 + EXCEPT + SELECT concept_id FROM concept_set_members WHERE codeset_id = :cset_2_codeset_id + """, {'codeset_id_1': codeset_id_1, 'cset_2_codeset_id': codeset_id_2}) + + added = sql_query_single_col(con, """ + SELECT concept_id FROM concept_set_members WHERE codeset_id = :cset_2_codeset_id + EXCEPT + SELECT concept_id FROM concept_set_members WHERE codeset_id = :codeset_id_1 + """, {'codeset_id_1': codeset_id_1, 'cset_2_codeset_id': codeset_id_2}) + flag_cnts_1 = ', flags: ' + ', '.join([f'{k}: {v}' for k, v in cset_1['flag_cnts'].items()]) if cset_1['flag_cnts'] else '' flag_cnts_2 = ', flags: ' + ', '.join([f'{k}: {v}' for k, v in cset_2['flag_cnts'].items()]) if cset_2['flag_cnts'] else '' @@ -628,20 +636,24 @@ def get_comparison_rpt(con, codeset_id_1: int, codeset_id_2: int) -> Dict[str, U 'name': cset_1['concept_set_name'], 'cset_1': f"{cset_1['codeset_id']} v{cset_1['version']}, " f"vocab {cset_1['omop_vocab_version']}; " - f"{cset_1['distinct_person_cnt']} pts, " - f"{cset_1['concepts']} concepts{flag_cnts_1}", + f"{commify(cset_1['distinct_person_cnt'])} pts, " + f"{commify(cset_1['total_cnt'] or cset_1['total_cnt_from_term_usage'])} recs, " + f"{commify(cset_1['concepts'])} concepts{flag_cnts_1}", 'cset_2': f"{cset_2['codeset_id']} v{cset_2['version']}, " f"vocab {cset_2['omop_vocab_version']}; " - f"{cset_2['distinct_person_cnt']} pts, " - f"{cset_2['concepts']} concepts{flag_cnts_2}", + f"{commify(cset_2['distinct_person_cnt'])} pts, " + f"{commify(cset_2['total_cnt'] or cset_2['total_cnt_from_term_usage'])} recs, " + f"{commify(cset_2['concepts'])} concepts{flag_cnts_2}", 'author': cset_1['codeset_creator'], 'cset_1_codeset_id': codeset_id_1, # 'cset_1_version': cset_1['version'], 'cset_2_codeset_id': codeset_id_2, + 'added': added, + 'removed': removed, # 'cset_2_version': cset_2['version'], # 'cset_1_only': cset_1_only, # 'cset_2_only': cset_2_only, - 'diffs': diffs, + 'diffs': diffs, # remove once front end working with new added/removed data } return rpt @@ -654,7 +666,7 @@ def generate_n3c_comparison_rpt(): """ SELECT orig_codeset_id, new_codeset_id FROM public.codeset_comparison - WHERE rpt IS NULL + --WHERE rpt IS NULL """) i = 1 for pair in pairs: @@ -691,5 +703,5 @@ def next_api_call_group_id() -> int: if __name__ == '__main__': from backend.utils import pdump # n3c_comparison_rpt() - # generate_n3c_comparison_rpt() + generate_n3c_comparison_rpt() pass \ No newline at end of file diff --git a/frontend/src/App.css b/frontend/src/App.css index 9c4eb7ce9..d2daa66b3 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -244,6 +244,28 @@ textarea { 100% { transform: rotate(359deg); } } +/* for n3c comparison diff tables */ +#n3ccompdiff { + font-family: Arial, Helvetica, sans-serif; + border-collapse: collapse; + /* width: 100%;*/ +} + +#n3ccompdiff td, #n3ccompdiff th { + border: 1px solid #ddd; + padding: 2px 5px 2px 5px; +} +#n3ccompdiff tr:nth-child(even){background-color: #f2f2f2;} + +#n3ccompdiff tr:hover {background-color: #ddd;} + +#n3ccompdiff th { + padding-top: 12px; + padding-bottom: 12px; + text-align: left; + background-color: #04AA6D; + color: white; +} diff --git a/frontend/src/components/ConceptSetCard.jsx b/frontend/src/components/ConceptSetCard.jsx index 9bad26048..11d99d1b4 100644 --- a/frontend/src/components/ConceptSetCard.jsx +++ b/frontend/src/components/ConceptSetCard.jsx @@ -131,6 +131,8 @@ export function ConceptSetCard(props) { ? "~ " + cset.distinct_person_cnt.toLocaleString() : ''; display_props["Record count"] = typeof(cset.total_cnt) === 'number' ? "~ " + cset.total_cnt.toLocaleString() : ''; + display_props["Record count from term usage"] = typeof(cset.total_cnt_from_term_usage) === 'number' + ? "~ " + cset.total_cnt_from_term_usage.toLocaleString() : ''; if (cset.is_most_recent_version) { tags.push("Most recent version"); diff --git a/frontend/src/components/CsetComparisonPage.jsx b/frontend/src/components/CsetComparisonPage.jsx index 67d682588..105c09d91 100644 --- a/frontend/src/components/CsetComparisonPage.jsx +++ b/frontend/src/components/CsetComparisonPage.jsx @@ -696,6 +696,13 @@ function colConfig(props) { width: 100, style: { justifyContent: "center" }, }, + { + name: "Std", + selector: (row) => row.standard_concept, + sortable: !nested, + width: 30, + style: { justifyContent: "center" }, + }, { name: "Concept ID", selector: (row) => row.concept_id < 0 ? '' : row.concept_id, diff --git a/frontend/src/components/N3CRecommended.jsx b/frontend/src/components/N3CRecommended.jsx index 1a7d63765..6593c42e7 100644 --- a/frontend/src/components/N3CRecommended.jsx +++ b/frontend/src/components/N3CRecommended.jsx @@ -1,5 +1,6 @@ import React, { useState, useEffect, } from "react"; import DataTable, { createTheme } from "react-data-table-component"; +import { flatten, uniq } from "lodash"; import {useDataGetter} from "../state/DataGetter"; import {useSearchParamsState} from "../state/SearchParamsProvider"; @@ -92,7 +93,9 @@ export const N3CComparisonRpt = () => { } try { const rows = await dataGetter.axiosCall('n3c-comparison-rpt', {sendAlert: false, }); - setData(rows); + let concept_ids = uniq(flatten(rows.map(row => [...(row.added), ...(row.removed)]))); + const concepts = await dataGetter.fetchAndCacheItems(dataGetter.apiCalls.concepts, concept_ids); + setData({rows, concepts}); } catch (error) { console.error("Error fetching data:", error); } @@ -102,16 +105,38 @@ export const N3CComparisonRpt = () => { if (!data) { return
Loading...
; } - let columns; - /* - if (data) { - columns = Object.keys(data[0]).map(col => ({ - name: col, - selector: row => (row[col] ?? '').toString(), - // width: "fit-content", - })); + let {rows, concepts} = data + function tbl(concept_ids) { + return ( + { + concept_ids.map((concept_id,i) => { + const c = concepts[concept_id]; + return ( + + + + + ) + }) + }
{c.concept_id}{c.standard_concept === 'S' ? 'Standard' : c.standard_concept === 'C' ? 'Classification' : 'Non-standard'}{c.concept_name}
+ ) } - */ + + function DiffList({data: row}) { + console.log({row}); + return ( +
+

+ Removed:{tbl(row.removed)} +

+

+ Added:{tbl(row.added)} +

+
+ ); + } + + let columns; columns = [ {grow: 4, sortable: true, name: "Name", selector: row => row.name}, {grow: 2, sortable: true, name: "Author", selector: row => row.author}, @@ -129,26 +154,12 @@ export const N3CComparisonRpt = () => { )}, - /* - {grow: 2, name: "Orig", selector: row => ( - - {row.orig_codeset_id} v{row.orig_version} {' '} - {} - - )}, - {grow: 2, name: "New", selector: row => ( - )}, - */ ] - - /* - */ - console.log({columns, data}); return (
{
); } -function DiffList({data}) { - console.log({data}); - return ( -
-

- Differences:
- { - data.diffs.map((diff,i) => ( - {diff}
 - )) - } -

-
- ); -}