Merging staging branch into prod branch

backend/cellguide/pipeline/__init__.py

@@ -17,6 +17,12 @@
 from backend.cellguide.pipeline.ontology_tree import run as run_ontology_tree_pipeline
 from backend.cellguide.pipeline.source_collections import run as run_source_collections_pipeline
 from backend.common.utils.cloudfront import create_invalidation_for_cellguide_data
+from backend.common.utils.result_notification import (
+    format_failed_batch_issue_slack_alert,
+    gen_cg_pipeline_failure_message,
+    gen_cg_pipeline_success_message,
+    notify_slack,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -55,8 +61,8 @@ def run_cellguide_pipeline():
         gpt_seo_output_directory=GPT_SEO_OUTPUT_DIRECTORY_FOLDERNAME,
     )
 
-    upload_cellguide_pipeline_output_to_s3(output_directory=output_directory)
-    upload_gpt_descriptions_to_s3(
+    output_path = upload_cellguide_pipeline_output_to_s3(output_directory=output_directory)
+    description_output_path = upload_gpt_descriptions_to_s3(
         gpt_output_directory=GPT_OUTPUT_DIRECTORY_FOLDERNAME,
         gpt_seo_output_directory=GPT_SEO_OUTPUT_DIRECTORY_FOLDERNAME,
     )
@@ -67,6 +73,8 @@ def run_cellguide_pipeline():
     # cleanup
     cleanup(output_directory=output_directory)
 
+    return output_path, description_output_path
+
 
 def upload_cellguide_pipeline_output_to_s3(*, output_directory: str):
     """
@@ -97,6 +105,10 @@ def upload_cellguide_pipeline_output_to_s3(*, output_directory: str):
     # this is used for custom cloudfront error handling
     s3_provider.upload_file("404", bucket, "404", {})
 
+    output_path = f"{bucket_path}{output_directory}"
+
+    return output_path
+
 
 def upload_gpt_descriptions_to_s3(*, gpt_output_directory: str, gpt_seo_output_directory: str) -> None:
     bucket_path = get_bucket_path()
@@ -113,6 +125,10 @@ def upload_gpt_descriptions_to_s3(*, gpt_output_directory: str, gpt_seo_output_d
         num_descriptions = len(glob(f"{src_directory}/*.json"))
         logger.info(f"Uploaded {num_descriptions} GPT descriptions to {bucket_path}{dst_directory}/")
 
+    description_output_path = f"{bucket_path}{dst_directory}/"
+
+    return description_output_path
+
 
 def cleanup(*, output_directory: str):
     logger.info(f"Cleaning up {output_directory} and other CellGuide pipeline outputs")
@@ -129,5 +145,14 @@ def cleanup(*, output_directory: str):
 
 
 if __name__ == "__main__":
-    run_cellguide_pipeline()
+    try:
+        output_path, description_output_path = run_cellguide_pipeline()
+        success_message = gen_cg_pipeline_success_message(output_path, description_output_path)
+        notify_slack(success_message)
+    except Exception as e:
+        logger.exception("Cell Guide Pipeline failed")
+        failure_message = format_failed_batch_issue_slack_alert(
+            gen_cg_pipeline_failure_message(f"Issue with Cell Guide pipeline run: {e}. See logs for more detail.")
+        )
+        notify_slack(failure_message)
     sys.exit()

backend/common/utils/result_notification.py

@@ -106,3 +106,48 @@ def gen_wmg_pipeline_success_message(snapshot_path: str, dataset_count: int, cel
             },
         ]
     }
+
+
+def gen_cg_pipeline_success_message(output_path: str, description_output_path: str) -> dict:
+    return {
+        "blocks": [
+            {
+                "type": "header",
+                "text": {
+                    "type": "plain_text",
+                    "text": "CellGuide Pipeline Run Succeeded:tada: ",
+                    "emoji": True,
+                },
+            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"\n* CellGuide snapshot stored in {output_path}"
+                    f"\n* GPT Descriptions can be found in {description_output_path}.",
+                },
+            },
+        ]
+    }
+
+
+def gen_cg_pipeline_failure_message(failure_info: str) -> dict:
+    return {
+        "blocks": [
+            {
+                "type": "header",
+                "text": {
+                    "type": "plain_text",
+                    "text": "CellGuide Pipeline Run FAILED:alert:",
+                    "emoji": True,
+                },
+            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"CellGuide Pipeline failure @sc-oncall-eng \n{failure_info}",
+                },
+            },
+        ]
+    }

backend/layers/processing/upload_failures/app.py

@@ -138,7 +138,7 @@ def get_failure_slack_notification_message(
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": f"Dataset processing job failed! @sc-oncall-eng please follow the [triage steps](https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y)\n"
+                    "text": f"Dataset processing job failed! Please follow the triage steps: https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y\n"
                     f"*Owner*: {collection_owner}\n"
                     f"*Collection URL*: {collection_url}\n"
                     f"*Collection Version URL*: {collection_version_url}\n"

frontend/src/views/CellGuide/components/CellGuideCard/components/Description/constants.ts

@@ -12,3 +12,9 @@ export const CELL_GUIDE_CARD_SYNONYMS = "cell-guide-card-synonyms";
 
 export const CELL_GUIDE_CARD_VALIDATED_DESCRIPTION =
   "cell-guide-card-validated-description";
+
+export const CELL_GUIDE_CARD_DEFAULT_DESCRIPTION_CL =
+  "Description not available";
+
+export const getDefaultGptDescription = (cellTypeName: string) =>
+  `Description for ${cellTypeName} is not available at the moment, please check back at a later time, or click on the link below for more information.`;

frontend/src/views/CellGuide/components/CellGuideCard/components/Description/index.tsx

@@ -47,6 +47,8 @@ import {
   CELL_GUIDE_CARD_GPT_TOOLTIP_LINK,
   DESCRIPTION_BREAKPOINT_HEIGHT_PX,
   CELL_GUIDE_CARD_SYNONYMS,
+  getDefaultGptDescription,
+  CELL_GUIDE_CARD_DEFAULT_DESCRIPTION_CL,
 } from "src/views/CellGuide/components/CellGuideCard/components/Description/constants";
 import { useIsComponentPastBreakpointHeight } from "../common/hooks/useIsComponentPastBreakpoint";
 import { StyledQuestionMarkIcon } from "src/common/style";
@@ -124,11 +126,14 @@ export default function Description({
     }
   }, [isPastBreakpoint]);
 
-  const { data: rawDescriptionGpt } = useGptDescription(cellTypeId);
+  const { data: rawDescriptionGpt = getDefaultGptDescription(cellTypeName) } =
+    useGptDescription(cellTypeId);
   const { data: rawDescriptionValidated, isLoading } =
     useValidatedDescription(cellTypeId);
   const { data: cellTypesById } = useCellTypeMetadata();
-  const rawDescriptionCl = cellTypesById?.[cellTypeId].clDescription;
+  const rawDescriptionCl =
+    cellTypesById?.[cellTypeId]?.clDescription ||
+    CELL_GUIDE_CARD_DEFAULT_DESCRIPTION_CL;
 
   useEffect(() => {
     if (rawDescriptionValidated) {

frontend/src/views/WheresMyGeneV2/components/CellInfoSideBar/index.tsx

@@ -61,6 +61,7 @@ import {
 import Description from "src/views/CellGuide/components/CellGuideCard/components/Description";
 import { StyledQuestionMarkIcon } from "src/common/style";
 import { DIFFERENTIAL_EXPRESSION_RELEASED_FLAG } from "src/views/DifferentialExpression/common/constants";
+import { useGptDescription } from "src/common/queries/cellGuide";
 
 function CellInfoSideBar({
   cellInfoCellType,
@@ -81,6 +82,10 @@ function CellInfoSideBar({
     cellInfoCellType,
   });
 
+  const { data: rawDescriptionGpt } = useGptDescription(
+    cellInfoCellType.cellType.id
+  );
+
   if (isLoading || !data) return null;
 
   if (!cellInfoCellType) return null;
@@ -100,19 +105,21 @@ function CellInfoSideBar({
         skinnyMode={true}
         inSideBar
       />
-      <StyledLink
-        href={`${ROUTES.CELL_GUIDE}/${cellInfoCellType.cellType.id}`}
-        onClick={() =>
-          track(EVENTS.WMG_OPEN_IN_CG_CLICKED, {
-            cell_type: cellInfoCellType.cellType.id,
-          })
-        }
-        target="_blank"
-        rel="noreferrer noopener"
-      >
-        {MARKER_SCORE_CELLGUIDE_LINK_TEXT}
-        <Icon sdsIcon="ChevronRight" sdsType="static" sdsSize="xs" />
-      </StyledLink>
+      {Boolean(rawDescriptionGpt) && (
+        <StyledLink
+          href={`${ROUTES.CELL_GUIDE}/${cellInfoCellType.cellType.id}`}
+          onClick={() =>
+            track(EVENTS.WMG_OPEN_IN_CG_CLICKED, {
+              cell_type: cellInfoCellType.cellType.id,
+            })
+          }
+          target="_blank"
+          rel="noreferrer noopener"
+        >
+          {MARKER_SCORE_CELLGUIDE_LINK_TEXT}
+          <Icon sdsIcon="ChevronRight" sdsType="static" sdsSize="xs" />
+        </StyledLink>
+      )}
       {DIFFERENTIAL_EXPRESSION_RELEASED_FLAG && (
         <StyledLink
           href={differentialExpressionUrl}

frontend/tests/features/wheresMyGene/tissueAutoExpand.test.ts

@@ -16,7 +16,7 @@ import {
   WMG_WITH_SEEDED_GENES_AND_TISSUES,
 } from "tests/utils/wmgUtils";
 
-const FILTERED_TISSUES = ["axilla", "blood", "brain"];
+const FILTERED_TISSUES = ["blood", "brain", "liver"];
 const TISSUE_NODE_TEST_ID = "tissue-name";
 const TISSUE_FILTER_LABEL = "Tissue";
 
@@ -34,8 +34,8 @@ const SELF_REPORTED_ETHNICITY_TISSUE = ["brain", "breast"];
 const DISEASE_FILTER_LABEL = "Disease";
 const DISEASE_FILTER_SELECTION = "influenza";
 
-const EXPECTED_FILTERED_TISSUES_WITH_SEX_FILTER = ["blood", "brain"];
-const EXPECTED_EXPANDED_TISSUES = ["blood"];
+const EXPECTED_FILTERED_TISSUES_WITH_SEX_FILTER = ["blood", "brain", "liver"];
+const EXPECTED_EXPANDED_TISSUES = ["blood", "liver"];
 const EXPECTED_VISIBLE_CELL = ["B Cell"];
 const EXPECTED_FILTERED_TISSUES_WITH_DISEASE_FILTER = ["blood", "brain"];
 const EXPECTED_FILTERED_TISSUES_WITH_B_CELL_FILTER = ["blood"];
@@ -169,19 +169,13 @@ describe("WMG tissue auto-expand", () => {
 
   /**
    * Tissue auto expansion - cross filter with Sex filter, check expansion
-   * Filter 3 Tissues. Collapse Abdomen. Select
-   * Female from the Sex filter. Tissue filter should now only have Abdomen and
-   * Blood selected. Only Abdomen and Blood should be visible. Abdomen should
-   * remain collapsed. Remove Sex filter. Tissue filter should now only have
-   * Abdomen and Blood selected. Only Abdomen and Blood should be visible.
-   * Abdomen should remain collapsed.
    */
   test("Tissue auto expansion - cross filter with Sex filter, check expansion", async ({
     page,
   }) => {
     await loadPageAndTissues(page);
     await filterTissues(page, FILTERED_TISSUES);
-    await collapseTissue(page, FILTERED_TISSUES[0]);
+    await collapseTissue(page, FILTERED_TISSUES[1]);
     await filterSelection({
       page,
       filterTestId: SEX_FILTER_TEST_ID,

tests/unit/processing/test_handle_error.py

@@ -268,7 +268,7 @@ def test_get_failure_slack_notification_message_with_dataset_version_id_none(
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": f"Dataset processing job failed! @sc-oncall-eng please follow the [triage steps](https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y)\n"
+                    "text": f"Dataset processing job failed! Please follow the triage steps: https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y\n"
                     "*Owner*: \n"
                     f"*Collection URL*: https://cellxgene.cziscience.com/collections/collection123\n"
                     f"*Collection Version URL*: https://cellxgene.cziscience.com/collections/{collection_version_id}\n"
@@ -320,7 +320,7 @@ def test_get_failure_slack_notification_message_with_dataset_not_found(
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": f"Dataset processing job failed! @sc-oncall-eng please follow the [triage steps](https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y)\n"
+                    "text": f"Dataset processing job failed! Please follow the triage steps: https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y\n"
                     "*Owner*: \n"
                     f"*Collection URL*: https://cellxgene.cziscience.com/collections/collection123\n"
                     f"*Collection Version URL*: https://cellxgene.cziscience.com/collections/{collection_version_id}\n"
@@ -386,7 +386,7 @@ def test_get_failure_slack_notification_message_with_missing_collection(
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": f"Dataset processing job failed! @sc-oncall-eng please follow the [triage steps](https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y)\n"
+                    "text": f"Dataset processing job failed! Please follow the triage steps: https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y\n"
                     f"*Owner*: \n"
                     f"*Collection URL*: https://cellxgene.cziscience.com/collections/{collection_id}\n"
                     f"*Collection Version URL*: https://cellxgene.cziscience.com/collections/{collection_version_id}\n"
@@ -446,7 +446,7 @@ def test_get_failure_slack_notification_message_with_dataset_and_collection(
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": f"Dataset processing job failed! @sc-oncall-eng please follow the [triage steps](https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y)\n"
+                    "text": f"Dataset processing job failed! Please follow the triage steps: https://docs.google.com/document/d/1n5cngEIz-Lqk9737zz3makXGTMrEKT5kN4lsofXPRso/edit#bookmark=id.3ofm47y0709y\n"
                     f"*Owner*: {owner}\n"
                     f"*Collection URL*: https://cellxgene.cziscience.com/collections/{collection_id}\n"
                     f"*Collection Version URL*: https://cellxgene.cziscience.com/collections/{collection_version_id}\n"