This should finish #673

- Added total_cnt_from_term_usage to all_csets - Commented out unneeded parameters in _get_cset_members_items - Improved get_comparison_rpt but left old diff stuff so it doesn't break prod/dev until new front end code is deployed. - Added std concept as column to main comparison table and to N3C comparison report diff list
jhu-bids · Jan 17, 2024 · 75da915 · 75da915
1 parent 92ec28a
commit 75da915
Show file tree

Hide file tree

Showing 6 changed files with 117 additions and 65 deletions.
diff --git a/backend/db/ddl-11-all_csets.jinja.sql b/backend/db/ddl-11-all_csets.jinja.sql
@@ -1,6 +1,15 @@
 -- Table: all_csets ----------------------------------------------------------------------------------------------------
 DROP TABLE IF EXISTS {{schema}}all_csets{{optional_suffix}} CASCADE;
 
+CREATE TABLE {{schema}}cset_term_usage_rec_counts{{optional_suffix}} AS
+    SELECT csm.codeset_id, SUM(cwc.total_cnt) AS total_cnt
+    FROM {{schema}}concept_set_members csm
+    JOIN {{schema}}concepts_with_counts cwc ON csm.concept_id = cwc.concept_id
+    WHERE cwc.total_cnt > 0
+    GROUP BY csm.codeset_id;
+
+CREATE INDEX ctu_idx1{{optional_index_suffix}} ON {{schema}}cset_term_usage_rec_counts{{optional_suffix}}(codeset_id);
+
 CREATE TABLE {{schema}}all_csets{{optional_suffix}} AS
 -- table instead of view for performance (no materialized views in mySQL)
 -- TODO: but now we're on postgres should it be a materialized view?
@@ -45,13 +54,15 @@ WITH ac AS (SELECT DISTINCT cs.codeset_id,
                             -- COALESCE(members.concepts, 0) AS members,
                             -- COALESCE(items.concepts, 0) AS items,
                             COALESCE(cscc.approx_distinct_person_count, 0) AS distinct_person_cnt,
-                            COALESCE(cscc.approx_total_record_count, 0)    AS total_cnt
+                            COALESCE(cscc.approx_total_record_count, 0)    AS total_cnt,
+                            COALESCE(ctu.total_cnt, 0)                     AS total_cnt_from_term_usage
             FROM {{schema}}code_sets cs
                      LEFT JOIN {{schema}}OMOPConceptSet ocs
             ON cs.codeset_id = ocs."codesetId" -- need quotes because of caps in colname
                 JOIN {{schema}}concept_set_container csc ON cs.concept_set_name = csc.concept_set_name
                 LEFT JOIN {{schema}}omopconceptsetcontainer ocsc ON csc.concept_set_id = ocsc."conceptSetId"
                 LEFT JOIN {{schema}}concept_set_counts_clamped cscc ON cs.codeset_id = cscc.codeset_id
+                LEFT JOIN {{schema}}cset_term_usage_rec_counts ctu ON cs.codeset_id = ctu.codeset_id
 /*
  want to add term usage record counts, tried this code:
      COALESCE(cwc.total_cnt, 0)                     AS total_cnt_from_term_usage
@@ -75,4 +86,6 @@ LEFT JOIN {{schema}}researcher rver ON ac.codeset_created_by = rver."multipassId
 
 CREATE INDEX ac_idx1{{optional_index_suffix}} ON {{schema}}all_csets{{optional_suffix}}(codeset_id);
 
-CREATE INDEX ac_idx2{{optional_index_suffix}} ON {{schema}}all_csets{{optional_suffix}}(concept_set_name);
+CREATE INDEX ac_idx2{{optional_index_suffix}} ON {{schema}}all_csets{{optional_suffix}}(concept_set_name);
+
+DROP TABLE {{schema}}cset_term_usage_rec_counts{{optional_suffix}};
diff --git a/backend/routes/db.py b/backend/routes/db.py
@@ -13,7 +13,7 @@
 from starlette.responses import Response
 
 from backend.api_logger import Api_logger
-from backend.utils import get_timer, return_err_with_trace
+from backend.utils import get_timer, return_err_with_trace, commify
 from backend.db.utils import get_db_connection, sql_query, SCHEMA, sql_query_single_col, sql_in, run_sql
 from backend.db.queries import get_concepts
 from enclave_wrangler.objects_api import get_n3c_recommended_csets, enclave_api_call_caller, \
@@ -51,19 +51,17 @@
 #       probably don't need precision etc.
 #       switched _container suffix on duplicate col names to container_ prefix
 #       joined OMOPConceptSet in the all_csets ddl to get `rid`
-def get_csets(codeset_ids: List[int], con: Connection = None) -> List[Dict]:
+def get_csets(codeset_ids: List[int]) -> List[Dict]:
     """Get information about concept sets the user has selected"""
-    conn = con if con else get_db_connection()
-    rows: List = sql_query(
-        conn, """
-          SELECT *
-          FROM all_csets
-          WHERE codeset_id = ANY(:codeset_ids);""",
-        {'codeset_ids': codeset_ids})
+    with get_db_connection() as con:
+        rows: List = sql_query(
+            con, """
+              SELECT *
+              FROM all_csets
+              WHERE codeset_id = ANY(:codeset_ids);""",
+            {'codeset_ids': codeset_ids})
     # {'codeset_ids': ','.join([str(id) for id in requested_codeset_ids])})
     row_dicts = [dict(x) for x in rows]
-    if not con:
-        conn.close()
     for row in row_dicts:
         row['researchers'] = get_row_researcher_ids_dict(row)
 
@@ -137,16 +135,16 @@ def _cset_members_items(codeset_ids: Union[str, None] = Query(default=''), ) ->
 @router.get("/get-cset-members-items")
 async def _get_cset_members_items(request: Request,
                                   codeset_ids: str,
-                                  columns: Union[List[str], None] = Query(default=None),
-                                  column: Union[str, None] = Query(default=None),
+                                  # columns: Union[List[str], None] = Query(default=None),
+                                  # column: Union[str, None] = Query(default=None),
                                   # extra_concept_ids: Union[int, None] = Query(default=None)
                                   ) -> Union[List[int], List]:
     requested_codeset_ids = parse_codeset_ids(codeset_ids)
     rpt = Api_logger()
     await rpt.start_rpt(request, params={'codeset_ids': requested_codeset_ids})
 
     try:
-        rows = get_cset_members_items(requested_codeset_ids, columns, column)
+        rows = get_cset_members_items(requested_codeset_ids) #, columns, column
         await rpt.finish(rows=len(rows))
     except Exception as e:
         await rpt.log_error(e)
@@ -606,7 +604,6 @@ def get_comparison_rpt(con, codeset_id_1: int, codeset_id_2: int) -> Dict[str, U
             SELECT concept_id, concept_name FROM concept_set_members WHERE codeset_id = :cset_2_codeset_id
         ) x
     """, {'codeset_id_1': codeset_id_1, 'cset_2_codeset_id': codeset_id_2})
-    # orig_only = [dict(r) for r in orig_only]
     cset_1_only = [dict(r)['diff'] for r in cset_1_only]
 
     cset_2_only = sql_query(con, """
@@ -616,32 +613,47 @@ def get_comparison_rpt(con, codeset_id_1: int, codeset_id_2: int) -> Dict[str, U
             SELECT concept_id, concept_name FROM concept_set_members WHERE codeset_id = :codeset_id_1
         ) x
     """, {'codeset_id_1': codeset_id_1, 'cset_2_codeset_id': codeset_id_2})
-    # cset_2_only = [dict(r) for r in cset_2_only]
     cset_2_only = [dict(r)['diff'] for r in cset_2_only]
 
     diffs = cset_1_only + cset_2_only
 
+    removed = sql_query_single_col(con, """
+        SELECT concept_id FROM concept_set_members WHERE codeset_id = :codeset_id_1
+        EXCEPT
+        SELECT concept_id FROM concept_set_members WHERE codeset_id = :cset_2_codeset_id
+    """, {'codeset_id_1': codeset_id_1, 'cset_2_codeset_id': codeset_id_2})
+
+    added = sql_query_single_col(con, """
+        SELECT concept_id FROM concept_set_members WHERE codeset_id = :cset_2_codeset_id
+        EXCEPT
+        SELECT concept_id FROM concept_set_members WHERE codeset_id = :codeset_id_1
+    """, {'codeset_id_1': codeset_id_1, 'cset_2_codeset_id': codeset_id_2})
+
     flag_cnts_1 = ', flags: ' + ', '.join([f'{k}: {v}' for k, v in cset_1['flag_cnts'].items()]) if  cset_1['flag_cnts'] else ''
     flag_cnts_2 = ', flags: ' + ', '.join([f'{k}: {v}' for k, v in cset_2['flag_cnts'].items()]) if  cset_2['flag_cnts'] else ''
 
     rpt = {
         'name': cset_1['concept_set_name'],
         'cset_1': f"{cset_1['codeset_id']} v{cset_1['version']}, "
                   f"vocab {cset_1['omop_vocab_version']}; "
-                  f"{cset_1['distinct_person_cnt']} pts, "
-                  f"{cset_1['concepts']} concepts{flag_cnts_1}",
+                  f"{commify(cset_1['distinct_person_cnt'])} pts, "
+                  f"{commify(cset_1['total_cnt'] or cset_1['total_cnt_from_term_usage'])} recs, "
+                  f"{commify(cset_1['concepts'])} concepts{flag_cnts_1}",
         'cset_2': f"{cset_2['codeset_id']} v{cset_2['version']}, "
                   f"vocab {cset_2['omop_vocab_version']}; "
-                  f"{cset_2['distinct_person_cnt']} pts, "
-                  f"{cset_2['concepts']} concepts{flag_cnts_2}",
+                  f"{commify(cset_2['distinct_person_cnt'])} pts, "
+                  f"{commify(cset_2['total_cnt'] or cset_2['total_cnt_from_term_usage'])} recs, "
+                  f"{commify(cset_2['concepts'])} concepts{flag_cnts_2}",
         'author': cset_1['codeset_creator'],
         'cset_1_codeset_id': codeset_id_1,
         # 'cset_1_version': cset_1['version'],
         'cset_2_codeset_id': codeset_id_2,
+        'added': added,
+        'removed': removed,
         # 'cset_2_version': cset_2['version'],
         # 'cset_1_only': cset_1_only,
         # 'cset_2_only': cset_2_only,
-        'diffs': diffs,
+        'diffs': diffs, # remove once front end working with new added/removed data
     }
     return rpt
 
@@ -654,7 +666,7 @@ def generate_n3c_comparison_rpt():
             """
                 SELECT orig_codeset_id, new_codeset_id
                 FROM public.codeset_comparison
-                WHERE rpt IS NULL
+                --WHERE rpt IS NULL
                 """)
         i = 1
         for pair in pairs:
@@ -691,5 +703,5 @@ def next_api_call_group_id() -> int:
 if __name__ == '__main__':
     from backend.utils import pdump
     # n3c_comparison_rpt()
-    # generate_n3c_comparison_rpt()
+    generate_n3c_comparison_rpt()
     pass
diff --git a/frontend/src/App.css b/frontend/src/App.css
@@ -244,6 +244,28 @@ textarea {
     100% { transform: rotate(359deg); }
 }
 
+/* for n3c comparison diff tables */
+#n3ccompdiff {
+    font-family: Arial, Helvetica, sans-serif;
+    border-collapse: collapse;
+    /* width: 100%;*/
+}
+
+#n3ccompdiff td, #n3ccompdiff th {
+    border: 1px solid #ddd;
+    padding: 2px 5px 2px 5px;
+}
 
+#n3ccompdiff tr:nth-child(even){background-color: #f2f2f2;}
+
+#n3ccompdiff tr:hover {background-color: #ddd;}
+
+#n3ccompdiff th {
+    padding-top: 12px;
+    padding-bottom: 12px;
+    text-align: left;
+    background-color: #04AA6D;
+    color: white;
+}
 
 
diff --git a/frontend/src/components/ConceptSetCard.jsx b/frontend/src/components/ConceptSetCard.jsx
@@ -131,6 +131,8 @@ export function ConceptSetCard(props) {
         ? "~ " + cset.distinct_person_cnt.toLocaleString() : '';
     display_props["Record count"] = typeof(cset.total_cnt) === 'number'
         ? "~ " + cset.total_cnt.toLocaleString() : '';
+    display_props["Record count from term usage"] = typeof(cset.total_cnt_from_term_usage) === 'number'
+        ? "~ " + cset.total_cnt_from_term_usage.toLocaleString() : '';
 
     if (cset.is_most_recent_version) {
       tags.push("Most recent version");

diff --git a/frontend/src/components/CsetComparisonPage.jsx b/frontend/src/components/CsetComparisonPage.jsx
@@ -696,6 +696,13 @@ function colConfig(props) {
       width: 100,
       style: { justifyContent: "center" },
     },
+    {
+      name: "Std",
+      selector: (row) => row.standard_concept,
+      sortable: !nested,
+      width: 30,
+      style: { justifyContent: "center" },
+    },
     {
       name: "Concept ID",
       selector: (row) => row.concept_id < 0 ? '' : row.concept_id,

diff --git a/frontend/src/components/N3CRecommended.jsx b/frontend/src/components/N3CRecommended.jsx
@@ -1,5 +1,6 @@
 import React, { useState, useEffect, } from "react";
 import DataTable, { createTheme } from "react-data-table-component";
+import { flatten, uniq } from "lodash";
 
 import {useDataGetter} from "../state/DataGetter";
 import {useSearchParamsState} from "../state/SearchParamsProvider";
@@ -92,7 +93,9 @@ export const N3CComparisonRpt = () => {
       }
       try {
         const rows = await dataGetter.axiosCall('n3c-comparison-rpt', {sendAlert: false, });
-        setData(rows);
+        let concept_ids = uniq(flatten(rows.map(row => [...(row.added), ...(row.removed)])));
+        const concepts = await dataGetter.fetchAndCacheItems(dataGetter.apiCalls.concepts, concept_ids);
+        setData({rows, concepts});
       } catch (error) {
         console.error("Error fetching data:", error);
       }
@@ -102,16 +105,38 @@ export const N3CComparisonRpt = () => {
   if (!data) {
     return <div>Loading...</div>;
   }
-  let columns;
-  /*
-  if (data) {
-    columns = Object.keys(data[0]).map(col => ({
-      name: col,
-      selector: row => (row[col] ?? '').toString(),
-      // width: "fit-content",
-    }));
+  let {rows, concepts} = data
+  function tbl(concept_ids) {
+    return (
+        <table id="n3ccompdiff"><tbody>{
+          concept_ids.map((concept_id,i) => {
+            const c = concepts[concept_id];
+            return (
+              <tr key={i}>
+                <td>{c.concept_id}</td>
+                <td><i>{c.standard_concept === 'S' ? 'Standard' : c.standard_concept === 'C' ? 'Classification' : 'Non-standard'}</i></td>
+                <td>{c.concept_name}</td>
+              </tr>)
+          })
+        }</tbody></table>
+    )
   }
-  */
+
+  function DiffList({data: row}) {
+    console.log({row});
+    return (
+        <div style={{margin: 10,}}>
+          <p>
+            <b>Removed:</b>{tbl(row.removed)}
+          </p>
+          <p>
+            <b>Added:</b>{tbl(row.added)}
+          </p>
+        </div>
+    );
+  }
+
+  let columns;
   columns = [
     {grow: 4, sortable: true, name: "Name", selector: row => row.name},
     {grow: 2, sortable: true, name: "Author", selector: row => row.author},
@@ -129,26 +154,12 @@ export const N3CComparisonRpt = () => {
           </Button>
 
       )},
-      /*
-    {grow: 2, name: "Orig", selector: row => (
-        <span>
-          {row.orig_codeset_id} v{row.orig_version} {' '}
-          {}
-        </span>
-      )},
-    {grow: 2, name: "New", selector: row => (
-      )},
-       */
   ]
-
-  /*
-   */
-
   console.log({columns, data});
   return (
       <div>
         <DataTable
-            data={data || []}
+            data={rows || []}
             columns={columns}
             expandableRows
             expandableRowsComponent={DiffList}
@@ -166,18 +177,3 @@ export const N3CComparisonRpt = () => {
       </div>
   );
 }
-function DiffList({data}) {
-  console.log({data});
-  return (
-      <div style={{margin: 10,}}>
-        <p>
-          <b>Differences:</b><br/>
-          {
-            data.diffs.map((diff,i) => (
-               <span key={i}>{diff}<br/></span>
-            ))
-          }
-        </p>
-      </div>
-  );
-}