FE-147 add sentry for production readiness interim dagster (#326)

* added Sentry to dependencies and Docker env * adding fetch depth 0 for sonar cloud * First attempt to integrate sentry - test 1 * Adding Sentry to our active pipelines * Adding config for pylint - ignoring check for unpacking parameters * Update requirements.txt --------- Co-authored-by: dsp-fieldeng-bot <[email protected]>
DataBiosphere · Jan 30, 2024 · d73a47e · d73a47e
1 parent 4ed2d88
commit d73a47e
Show file tree

Hide file tree

Showing 16 changed files with 753 additions and 11 deletions.
diff --git a/.github/linters/.pylintrc b/.github/linters/.pylintrc
diff --git a/.github/workflows/build_and_publish_dev.yaml b/.github/workflows/build_and_publish_dev.yaml
@@ -7,6 +7,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        # Needed by sonar to get the git history for the branch the PR will be merged into.
+        with:
+          fetch-depth: 0
       - name: Fetch tag history
         run: git fetch --tags
       - uses: google-github-actions/[email protected]

diff --git a/.github/workflows/build_and_publish_main.yaml b/.github/workflows/build_and_publish_main.yaml
@@ -1,14 +1,19 @@
 name: Main Validation and Release Workflow
 on:
   pull_request_target:
-      types:
-        - closed
+    types:
+      - closed
+    branches:
+      - main
 jobs:
   main-ci:
     if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        # Needed by sonar to get the git history for the branch the PR will be merged into.
+        with:
+          fetch-depth: 0
       - name: Fetch tag history
         run: git fetch --tags
       - uses: olafurpg/setup-scala@v10

diff --git a/.github/workflows/generate-requirements-file.yaml b/.github/workflows/generate-requirements-file.yaml
@@ -13,6 +13,9 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
+        # Needed by sonar to get the git history for the branch the PR will be merged into.
+        with:
+          fetch-depth: 0
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4

diff --git a/.github/workflows/super_linter.yaml b/.github/workflows/super_linter.yaml
@@ -59,4 +59,6 @@ jobs:
           VALIDATE_PYTHON_BLACK: false
           VALIDATE_PYTHON_MYPY: false
           DEFAULT_BRANCH: main
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LINTER_RULES_PATH: ../linters
+
diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml
@@ -8,6 +8,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        # Needed by sonar to get the git history for the branch the PR will be merged into.
+        with:
+          fetch-depth: 0
       - uses: broadinstitute/dsp-appsec-trivy-action@v1
         with:
              context: ./orchestration

diff --git a/.github/workflows/validate_pull_request_main.yaml b/.github/workflows/validate_pull_request_main.yaml
@@ -14,6 +14,7 @@ jobs:
           project_id: ${{ secrets.DEV_PROJECT_ID }}
           service_account_key: ${{ secrets.GCP_TEST_KEY }}
           export_default_credentials: true
+          fetch-depth: 0
       - name: Set up Python 3.9 for dataflow tests
         uses: actions/setup-python@v2
         with:

diff --git a/.github/workflows/validate_python.yaml b/.github/workflows/validate_python.yaml
@@ -15,6 +15,9 @@ jobs:
       ENV: test
     steps:
     - uses: actions/checkout@v2
+      # Needed by sonar to get the git history for the branch the PR will be merged into.
+      with:
+        fetch-depth: 0
     - uses: google-github-actions/[email protected]
       with:
         project_id: ${{ secrets.DEV_PROJECT_ID }}

diff --git a/orchestration/Dockerfile b/orchestration/Dockerfile
@@ -9,7 +9,8 @@ ENV PYTHONFAULTHANDLER=1 \
   PIP_NO_CACHE_DIR=off \
   PIP_DISABLE_PIP_VERSION_CHECK=on \
   PIP_DEFAULT_TIMEOUT=100 \
-  POETRY_VERSION=1.1.8
+  POETRY_VERSION=1.1.8 \
+  SENTRY_DSN=https://[email protected]/4506559533088768
 
 RUN pip install "poetry==$POETRY_VERSION"
 

diff --git a/orchestration/hca_orchestration/pipelines/cut_snapshot.py b/orchestration/hca_orchestration/pipelines/cut_snapshot.py
@@ -1,5 +1,7 @@
+import os
 import warnings
 
+import sentry_sdk
 from dagster import (
     ExperimentalWarning,
     HookContext,
@@ -36,6 +38,14 @@
 warnings.filterwarnings("ignore", category=ExperimentalWarning)
 
 
+SENTRY_DSN = os.getenv(
+    "SENTRY_DSN",
+    "",
+)
+if SENTRY_DSN:
+    sentry_sdk.init(dsn=SENTRY_DSN, traces_sample_rate=1.0)
+
+
 def cut_project_snapshot_job(hca_env: str, jade_env: str, steward: str) -> PipelineDefinition:
     return cut_snapshot.to_job(
         description="Given a source project ID, this pipeline will determine the corresponding "

diff --git a/orchestration/hca_orchestration/pipelines/load_hca.py b/orchestration/hca_orchestration/pipelines/load_hca.py
@@ -3,19 +3,41 @@
 area, transforming via Google Cloud Dataflow jobs into a form suitable for ingestion to TDR and the final
 load to TDR itself.
 """
+import os
 import warnings
 
-from dagster import graph, ExperimentalWarning
-
-from hca_orchestration.solids.load_hca.data_files.load_data_files import import_data_files
-from hca_orchestration.solids.load_hca.data_files.load_data_metadata_files import file_metadata_fanout
-from hca_orchestration.solids.load_hca.non_file_metadata.load_non_file_metadata import non_file_metadata_fanout
-from hca_orchestration.solids.load_hca.stage_data import clear_scratch_dir, pre_process_metadata, create_scratch_dataset
-from hca_orchestration.solids.load_hca.utilities import send_start_notification, validate_and_send_finish_notification
+import sentry_sdk
+from dagster import ExperimentalWarning, graph
+from hca_orchestration.solids.load_hca.data_files.load_data_files import (
+    import_data_files,
+)
+from hca_orchestration.solids.load_hca.data_files.load_data_metadata_files import (
+    file_metadata_fanout,
+)
+from hca_orchestration.solids.load_hca.non_file_metadata.load_non_file_metadata import (
+    non_file_metadata_fanout,
+)
+from hca_orchestration.solids.load_hca.stage_data import (
+    clear_scratch_dir,
+    create_scratch_dataset,
+    pre_process_metadata,
+)
+from hca_orchestration.solids.load_hca.utilities import (
+    send_start_notification,
+    validate_and_send_finish_notification,
+)
 
 warnings.filterwarnings("ignore", category=ExperimentalWarning)
 
 
+SENTRY_DSN = os.getenv(
+    "SENTRY_DSN",
+    "",
+)
+if SENTRY_DSN:
+    sentry_sdk.init(dsn=SENTRY_DSN, traces_sample_rate=1.0)
+
+
 @graph
 def load_hca() -> None:
     staging_dataset = create_scratch_dataset(

diff --git a/orchestration/hca_orchestration/pipelines/set_snapshot_public.py b/orchestration/hca_orchestration/pipelines/set_snapshot_public.py
@@ -1,5 +1,7 @@
+import os
 import warnings
 
+import sentry_sdk
 from dagster import (
     ExperimentalWarning,
     HookContext,
@@ -30,6 +32,13 @@
 
 warnings.filterwarnings("ignore", category=ExperimentalWarning)
 
+SENTRY_DSN = os.getenv(
+    "SENTRY_DSN",
+    "",
+)
+if SENTRY_DSN:
+    sentry_sdk.init(dsn=SENTRY_DSN, traces_sample_rate=1.0)
+
 
 def make_snapshot_public_job(hca_env: str, jade_env: str) -> PipelineDefinition:
     return set_snapshot_public.to_job(

diff --git a/orchestration/hca_orchestration/pipelines/validate_ingress.py b/orchestration/hca_orchestration/pipelines/validate_ingress.py
@@ -1,3 +1,6 @@
+import os
+
+import sentry_sdk
 from dagster import (
     HookContext,
     InitResourceContext,
@@ -13,6 +16,13 @@
     pre_flight_validate,
 )
 
+SENTRY_DSN = os.getenv(
+    "SENTRY_DSN",
+    "",
+)
+if SENTRY_DSN:
+    sentry_sdk.init(dsn=SENTRY_DSN, traces_sample_rate=1.0)
+
 
 def run_config_for_validation_ingress_partition(
     partition: Partition,

diff --git a/orchestration/poetry.lock b/orchestration/poetry.lock
diff --git a/orchestration/pyproject.toml b/orchestration/pyproject.toml
@@ -31,6 +31,7 @@ protobuf = "3.20.2"
 python-dateutil = "^2.8.1"
 pyyaml = "^5.3"
 rfc3339-validator = "^0.1.4"
+sentry-sdk = "^1.39.2"
 typing-extensions = "^3.7.4"
 # werkzeug = "2.2.3"
 # will have to update dagit which means updating broad-dagster-utils - FE-36

diff --git a/orchestration/requirements.txt b/orchestration/requirements.txt
@@ -80,6 +80,7 @@ rfc3339-validator==0.1.4; (python_version >= "2.7" and python_full_version < "3.
 rpds-py==0.10.6; python_version >= "3.8"
 rsa==4.9; python_version >= "3.6" and python_version < "4" and (python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_full_version >= "3.6.0" and python_version >= "3.9" and python_version < "3.10")
 rx==1.6.3; python_version >= "3.9" and python_version < "3.10"
+sentry-sdk==1.39.2
 six==1.16.0; python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_version >= "3.9" and python_version < "3.10" and python_full_version >= "3.6.0"
 slack-sdk==3.23.0; python_full_version >= "3.6.0"
 slackclient==2.9.4; python_version >= "3.9" and python_version < "3.10" and python_full_version >= "3.6.0"