Merge pull request #949 from jhu-bids/granular-caching

Granular caching

.github/workflows/frontend_dev.yml

@@ -17,9 +17,16 @@ jobs:
     runs-on: ubuntu-latest
     name: Build and Deploy
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22.x'
+          cache: 'yarn'
+
       - name: Print commit hash & branch for rollbacks & troubleshooting
         run: |
           echo "Commit hash: ${{ github.sha }}"

.github/workflows/frontend_prod.yml

@@ -20,6 +20,13 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22.x'
+          cache: 'yarn'
+
       - name: Print commit hash & branch for rollbacks & troubleshooting
         run: |
           echo "Commit hash: ${{ github.sha }}"

.github/workflows/test_frontend_e2e_live_dev.yml

@@ -16,12 +16,15 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: dev
-    - uses: actions/setup-node@v3
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
       with:
-        node-version: 18
+        node-version: '22.x'
+        cache: 'yarn'
 
     # Yarn Setup
     # cache: 'yarn' # Enable yarn cache

.github/workflows/test_frontend_e2e_live_dev_running_local.yml

@@ -17,12 +17,15 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: develop
-    - uses: actions/setup-node@v3
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
       with:
-        node-version: 18
+        node-version: '22.x'
+        cache: 'yarn'
 
     # Yarn Setup
     # cache: 'yarn' # Enable yarn cache

.github/workflows/test_frontend_e2e_live_prod.yml

@@ -16,15 +16,18 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         # TODO: If fails on prod, try dev again
         #  Q: why is this 'dev' if it's for the test on prod? A: Because when we deployed to prod, our playwright tests hadn't been updated yet.
 #        ref: dev
         ref: prod
-    - uses: actions/setup-node@v3
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
       with:
-        node-version: 18
+        node-version: '22.x'
+        cache: 'yarn'
 
     # Yarn Setup
     # cache: 'yarn' # Enable yarn cache

.github/workflows/test_frontend_unit_and_qc.yml

@@ -10,28 +10,23 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 18
+    - uses: actions/checkout@v4
 
-    # Yarn Setup
-    # cache: 'yarn' # Enable yarn cache
-    # manual caching steps
-    - name: get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v3
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
       with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('frontend/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
+        node-version: '22.x'
+        cache: 'yarn'
+        cache-dependency-path: 'frontend/yarn.lock'
+
     - name: Install dependencies
-      # run: cd frontend && npm ci  # only playwright is necessary to install
-      run: yarn install --frozen-lockfile # Equivalent to npm ci
       working-directory: frontend
+      run: |
+        echo "Current directory:"
+        pwd
+        echo "Directory contents:"
+        ls -la
+        yarn install --frozen-lockfile
 
-    # Run tests
     - name: Run tests
-      run: make test-frontend-unit
+      run: make test-frontend-unit

backend/db/config.py

@@ -120,10 +120,8 @@ def get_pg_connect_url(local=False):
         'codeset_counts', 'researcher'
     ],
     'codeset_counts': ['members_items_summary'],
-    'codeset_ids_by_concept_id': ['cset_members_items'],
     'concept_ancestor_plus': ['concept_ancestor', 'concepts_with_counts'],
     'concept_graph': ['concept_ancestor'],
-    'concept_ids_by_codeset_id': ['cset_members_items'],
     'concept_relationship_plus': ['concept_relationship', 'concepts_with_counts'],
     'concepts_with_counts': ['concepts_with_counts_ungrouped'],
     'concepts_with_counts_ungrouped': ['concept', 'deidentified_term_usage_by_domain_clamped'],

backend/db/ddl-6-cset_members_items.jinja.sql

@@ -23,21 +23,25 @@ SELECT DISTINCT
         CASE WHEN "includeMapped" THEN 'M' ELSE '' END,
         CASE WHEN "isExcluded" THEN 'X' ELSE '' END
     ) AS flags,
+    NULLIF(concat_ws(',',
+        CASE WHEN "isExcluded" THEN 'isExcluded' ELSE NULL END,
+        CASE WHEN "includeDescendants" THEN 'includeDescendants' ELSE NULL END,
+        CASE WHEN "includeMapped" THEN 'includeMapped' ELSE NULL END
+    ), '') AS item_flags,
     "isExcluded",
     "includeDescendants",
     "includeMapped"
 FROM csvi
 LEFT JOIN {{schema}}concept_set_members csm ON csvi.codeset_id = csm.codeset_id AND csvi.concept_id = csm.concept_id;
 
-DROP TABLE csvi;
-
 CREATE TEMP TABLE csmi2 AS
 SELECT DISTINCT
     csm.codeset_id,
     csm.concept_id,
     true AS "csm",
     false AS "item",
     NULL::text AS flags,
+    NULL::text AS item_flags,
     NULL::bool AS "isExcluded",
     NULL::bool AS "includeDescendants",
     NULL::bool AS "includeMapped"
@@ -47,7 +51,9 @@ WHERE csmi1.concept_id IS NULL
 UNION
 SELECT * FROM csmi1;
 
-DROP TABLE csmi1;
+CREATE INDEX csmi_idx21 ON csmi2(codeset_id);
+
+CREATE INDEX csmi_idx22 ON csmi2(concept_id);
 
 DROP TABLE IF EXISTS {{schema}}cset_members_items{{optional_suffix}} CASCADE;
 
@@ -64,10 +70,15 @@ LEFT JOIN csmi2 ON cs.codeset_id = csmi2.codeset_id
 JOIN {{schema}}concept c ON csmi2.concept_id = c.concept_id
 WHERE csmi2.codeset_id IS NOT NULL;
 
-DROP TABLE csmi2;
-
 CREATE INDEX csmi_idx1{{optional_index_suffix}} ON {{schema}}cset_members_items{{optional_suffix}}(codeset_id);
 
 CREATE INDEX csmi_idx2{{optional_index_suffix}} ON {{schema}}cset_members_items{{optional_suffix}}(concept_id);
 
 CREATE INDEX csmi_idx3{{optional_index_suffix}} ON {{schema}}cset_members_items{{optional_suffix}}(codeset_id, concept_id);
+
+DROP TABLE csvi;
+
+DROP TABLE csmi1;
+
+DROP TABLE csmi2;
+

backend/routes/db.py

@@ -310,69 +310,6 @@ def next_api_call_group_id() -> Optional[int]:
     return id
 
 
-@router.post("/concept-ids-by-codeset-id")
-async def get_concept_ids_by_codeset_id_post(request: Request, codeset_ids: Union[List[int], None] = None) -> Dict:
-    """Route for get_concept_ids_by_codeset_id() via POST"""
-    print(codeset_ids)
-    return await get_concept_ids_by_codeset_id(request, codeset_ids)
-
-
-@router.get("/concept-ids-by-codeset-id")
-@return_err_with_trace
-async def get_concept_ids_by_codeset_id(
-    request: Request, codeset_ids: Union[List[str], None] = Query(...)
-) -> Dict[int, List[int]]:
-    """Get concept IDs by codeset id"""
-    if codeset_ids:
-        q = f"""
-              SELECT csids.codeset_id, COALESCE(cibc.concept_ids, ARRAY[]::integer[]) AS concept_ids
-              FROM (VALUES{",".join([f"({csid})" for csid in codeset_ids])}) AS csids(codeset_id)
-              LEFT JOIN concept_ids_by_codeset_id cibc ON csids.codeset_id = cibc.codeset_id"""
-    else:
-        q = f"""SELECT * FROM concept_ids_by_codeset_id"""
-
-    rpt = Api_logger()
-    await rpt.start_rpt(request, params={'codeset_ids': codeset_ids})
-
-    try:
-        with get_db_connection() as con:
-            rows: List = sql_query(con, q)
-        await rpt.finish(rows=len(rows))
-    except Exception as e:
-        await rpt.log_error(e)
-        raise e
-    return {r['codeset_id']: r['concept_ids'] for r in rows}
-
-
-@router.post("/codeset-ids-by-concept-id")
-@return_err_with_trace
-async def get_codeset_ids_by_concept_id_post(
-    request: Request, concept_ids: Union[List[int], None] = None
-) -> Dict:  # Dict[int, List[int]]
-    """Get Codeset IDs by concept ID"""
-    q = f"""
-          SELECT *
-          FROM codeset_ids_by_concept_id"""
-    q += f" WHERE concept_id {sql_in(concept_ids)}" if concept_ids else ""
-    rpt = Api_logger()
-    await rpt.start_rpt(request, params={'concept_ids': concept_ids})
-    try:
-        with get_db_connection() as con:
-            rows: List = sql_query(con, q)
-        await rpt.finish(rows=len(rows))
-    except Exception as e:
-        await rpt.log_error(e)
-        raise e
-
-    return {r['concept_id']: r['codeset_ids']  for r in rows}
-
-
-@router.get("/codeset-ids-by-concept-id")
-async def get_codeset_ids_by_concept_id(request: Request, concept_ids: Union[List[str], None] = Query(...)) -> Dict:
-    """Get Codeset IDs by concept ID"""
-    return await get_codeset_ids_by_concept_id_post(request, concept_ids)
-
-
 @router.post("/related-cset-concept-counts")
 def get_related_cset_concept_counts(concept_ids: List[int] = None, verbose=True) -> Dict:
     """Returns dict of codeset_id: count of included concepts"""

frontend/package.json

@@ -30,6 +30,7 @@
     "graphology-simple-path": "^0.2.0",
     "graphology-traversal": "^0.3.1",
     "jest-environment-jsdom": "^29.7.0",
+    "lru-cache": "^11.0.2",
     "lz-string": "^1.5.0",
     "match-sorter": "^6.3.1",
     "papaparse": "^5.4.1",
@@ -104,8 +105,8 @@
     "typescript-eslint": "^8.5.0"
   },
   "engines": {
-    "npm": ">=8.9.0",
-    "node": ">=18.2.0"
+    "npm": ">=10.8.3",
+    "node": ">=22.9.0"
   },
   "jshintConfig": {
     "esversion": 11

frontend/src/components/AboutPage.jsx

@@ -84,9 +84,9 @@ export function AboutPage() {
       console.log(
           'Triggering database refresh and clearing cache so new data will be fetched when ready');
       // empty cache immediately, and then again after the db-refresh call is done
-      dataCache.emptyCache();
+      dataCache.clear();
       await dataGetter.axiosCall('db-refresh');
-      dataCache.emptyCache();
+      dataCache.clear();
     } catch (error) {
       console.error('Error:', error);
     }
@@ -205,7 +205,7 @@ export function AboutPage() {
               variant={'contained'}
               // onClick={() => queryClient.removeQueries()}
               onClick={() => {
-                dataCache.emptyCache();
+                dataCache.clear();
                 if (loadCSetsRef.current) loadCSetsRef.current.value = '';
                 resetReducers();
                 // maybe resetReducers makes these redundant?

frontend/src/components/CsetComparisonPage.jsx

@@ -70,9 +70,6 @@ export async function fetchGraphData(props) {
     dataGetter.fetchAndCacheItems(dataGetter.apiCalls.csets, codeset_ids),
   ];
   // have to get concept_ids before fetching concepts
-  // const concept_ids_by_codeset_id = await dataGetter.fetchAndCacheItems(dataGetter.apiCalls.concept_ids_by_codeset_id, codeset_ids);
-  // let concept_ids = union(flatten(Object.values(concept_ids_by_codeset_id)));
-
   const graphData = await dataGetter.fetchAndCacheItems(
       dataGetter.apiCalls.concept_graph_new,
       {codeset_ids: codeset_ids || [], cids: cids || []});

frontend/src/components/Csets.jsx

@@ -269,17 +269,12 @@ export function ConceptSetsPage () {
 
       let selected_csets = dataGetter.fetchAndCacheItems(
         dataGetter.apiCalls.csets, codeset_ids);
-      // returnFunc: results => [...Object.values(results)]; // isn't this the same as shape: 'array'?
-      let concept_ids = dataGetter.fetchAndCacheItems(
-        dataGetter.apiCalls.concept_ids_by_codeset_id, codeset_ids);
-      // returnFunc: results => union(flatten(Object.values(results)))
 
-      concept_ids = await concept_ids;
-      concept_ids = uniq(flatten(Object.values(await concept_ids)));
-      // setData(current => ({...current, concept_ids}));
+      let csmi = await dataGetter.fetchAndCacheItems(dataGetter.apiCalls.cset_members_items, codeset_ids);
+
+      let concept_ids = flatten(Object.values(csmi).map(d => Object.values(d))).map(d => d.concept_id);
 
       all_csets = await all_csets;
-      // setData(current => ({...current, all_csets, }));
 
       let relatedCsetConceptCounts = await dataGetter.axiosCall(
           'related-cset-concept-counts',

frontend/src/state/AppState.jsx

@@ -483,7 +483,7 @@ State managers / reducer providers and their storage providers
       - cids
 
 
-2023-08
+2023-08     VERY OUT OF DATE
 State management is pretty messed up at the moment. We need decent performance....
 Here's what needs to be tracked in state and description of how it's all related.
 
@@ -562,7 +562,7 @@ export function ViewCurrentState () {
     <Inspector data={{ /*alerts, */ graphOptions, newCset }}/>
 
     <h2>dataCache</h2>
-    <Inspector data={dataCache.getWholeCache()}/>
+    <Inspector data={dataCache}/>
 
     <h2>The different kinds of state</h2>
     <Markdown>{stateDoc}</Markdown>