created separate step for datadog failure content-release notification

department-of-veterans-affairs · Oct 20, 2023 · 731039e · 731039e
1 parent 249c6e5
commit 731039e
Showing 1 changed file with 45 additions and 28 deletions.
diff --git a/.github/workflows/content-release.yml b/.github/workflows/content-release.yml
@@ -553,34 +553,6 @@ jobs:
           url: ${{ env.DRUPAL_ADDRESS }}/api/content_release/error
           method: GET
 
-      - name: Get Datadog token from Parameter Store
-        uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb # latest
-        with:
-          ssm_parameter: /dsva-vagov/content-build/GHA_CONTENT_BUILD_DATADOG_API_KEY
-          env_variable_name: GHA_CONTENT_BUILD_DATADOG_API_KEY
-
-      - name: Build JSON object
-        run: |
-          jq --null-input '{}' | \
-          jq '.title = "VA.gov CMS content release has failed"' | \
-          jq '.text = "VA.gov Content release ${{github.run_id}} failed at \(now|strftime("%Y-%m-%d %H:%M:%S"))! https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}"' | \
-          jq '.date_happened = now' | \
-          jq '.aggregation_key = "content release ${{github.run_id}}"' | \
-          jq '.tags[0] = "project:vagov"' | \
-          jq '.tags[1] = "repo:content-build"' | \
-          jq '.tags[2] = "workflow:content-release"' | \
-          jq '.tags[3] = "env:${{env.DEPLOY_ENV}}"' | \
-          jq '.tags[5] = "status:${{needs.deploy.result}}"' | \
-          jq '.tags[6] = "trigger:${{env.BUILD_TRIGGER}}"' | \
-          jq '.alert_type = "error"' > event.json
-
-      - name: Send event to Datadog
-        run: |
-          curl -X POST "https://api.ddog-gov.com/api/v1/events" \
-          -H "Content-Type: text/json" \
-          -H "DD-API-KEY: ${{ env.GHA_CONTENT_BUILD_DATADOG_API_KEY }}" \
-          -d @- < event.json
-
   notify-failure-slack:
     name: Notify Failure (Slack)
     runs-on: [self-hosted, asg]
@@ -613,6 +585,51 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
+  notify-failure-datadog:
+    name: Notify Failure (Datadog)
+    runs-on: [self-hosted, asg]
+    if: |
+      (failure() && needs.deploy.result != 'success')
+    needs: deploy
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-gov-west-1
+
+      - name: Get Datadog token from Parameter Store
+        uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb # latest
+        with:
+          ssm_parameter: /dsva-vagov/content-build/GHA_CONTENT_BUILD_DATADOG_API_KEY
+          env_variable_name: GHA_CONTENT_BUILD_DATADOG_API_KEY
+
+      - name: Build JSON object
+        run: |
+          jq --null-input '{}' | \
+          jq '.title = "VA.gov CMS content release has failed"' | \
+          jq '.text = "VA.gov Content release ${{github.run_id}} failed at \(now|strftime("%Y-%m-%d %H:%M:%S"))! https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}"' | \
+          jq '.date_happened = now' | \
+          jq '.aggregation_key = "content release ${{github.run_id}}"' | \
+          jq '.tags[0] = "project:vagov"' | \
+          jq '.tags[1] = "repo:content-build"' | \
+          jq '.tags[2] = "workflow:content-release"' | \
+          jq '.tags[3] = "env:${{env.DEPLOY_ENV}}"' | \
+          jq '.tags[5] = "status:${{needs.deploy.result}}"' | \
+          jq '.tags[6] = "trigger:${{env.BUILD_TRIGGER}}"' | \
+          jq '.alert_type = "error"' > event.json
+
+      - name: Send event to Datadog
+        run: |
+          curl -X POST "https://api.ddog-gov.com/api/v1/events" \
+          -H "Content-Type: text/json" \
+          -H "DD-API-KEY: ${{ env.GHA_CONTENT_BUILD_DATADOG_API_KEY }}" \
+          -d @- < event.json
+
   record-metrics:
     name: Record metrics in Datadog
     runs-on: [self-hosted, asg]