feat: crunchy deployment

.github/workflows/.dbdeployer.yml

@@ -0,0 +1,166 @@
+name: .Crunchy Deploy
+
+on:
+  workflow_call:
+    inputs: ### Required
+      directory:
+        description: Crunchy Chart directory
+        default: 'charts/crunchy'
+        required: false
+        type: string
+      oc_server:
+        default: https://api.silver.devops.gov.bc.ca:6443
+        description: 'OpenShift server'
+        required: false
+        type: string
+      environment:
+        description: Environment name; omit for PRs
+        required: false
+        type: string
+      s3_enabled:
+        description: Enable S3 backups
+        required: false
+        default: true
+        type: boolean
+      values:
+        description: 'Values file'
+        default: 'values.yaml'
+        required: false
+        type: string
+      app_values:
+        description: 'App specific values file which is present inside charts/app'
+        default: 'values.yaml'
+        required: false
+        type: string
+      enabled:
+        description: 'Enable the deployment of the crunchy database, easy switch to turn it on/off'
+        default: true
+        required: false
+        type: boolean
+      timeout-minutes:
+        description: 'Timeout minutes'
+        default: 20
+        required: false
+        type: number
+      triggers:
+        description: Paths used to trigger a database deployment
+        required: false
+        type: string
+    secrets:
+      oc_namespace:
+        description: OpenShift namespace
+        required: true
+      oc_token:
+        description: OpenShift token
+        required: true
+      s3_access_key:
+        description: S3 access key
+        required: false
+      s3_secret_key:
+        description: S3 secret key
+        required: false
+jobs:
+  deploy_db:
+    timeout-minutes: ${{ inputs.timeout-minutes }}
+    runs-on: ubuntu-24.04
+    if: ${{ inputs.enabled }}
+    name: Deploy Or Upgrade Crunchy DB
+    environment: ${{ inputs.environment }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install CLI tools from OpenShift Mirror
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14.37"
+      - uses: bcgov-nr/[email protected]
+        id: triggers
+        with:
+          triggers: ${{ inputs.triggers }}
+      - name: Validate Inputs
+        if: steps.triggers.outputs.triggered == 'true'
+        shell: bash
+        run: |
+          if [ ${{ inputs.s3_enabled }} == true ]; then
+            echo "S3 ie enabled for backups, checking for mandatory secrets"
+            if [ ! "${{ secrets.s3_access_key }}" ]; then
+              echo "S3 access key not found"
+              exit 1
+            fi
+            if [ ! "${{ secrets.s3_secret_key }}" ]; then
+              echo "S3 secret key not found"
+              exit 1
+            fi
+          fi
+
+      - name: OC Login
+        shell: bash
+        run: |
+          # OC Login
+          OC_TEMP_TOKEN=$(curl -k -X POST ${{ inputs.oc_server }}/api/v1/namespaces/${{ secrets.oc_namespace }}/serviceaccounts/pipeline/token --header "Authorization: Bearer ${{ secrets.oc_token }}" -d '{"spec": {"expirationSeconds": 600}}' -H 'Content-Type: application/json; charset=utf-8' | jq -r '.status.token' )
+
+          oc login --token=$OC_TEMP_TOKEN --server=${{ inputs.oc_server }}
+          oc project ${{ secrets.oc_namespace }} # Safeguard!
+
+      - name: Deploy Database
+        if: steps.triggers.outputs.triggered == 'true'
+        working-directory: ${{ inputs.directory }}
+        shell: bash
+        run: |
+          echo 'Deploying crunchy helm chart'
+          if [ ${{ inputs.s3_enabled }} == true ]; then
+            helm upgrade --install --wait --set crunchy.pgBackRest.s3.enabled=true \
+             --set-string crunchy.pgBackRest.s3.accessKey=${{ secrets.s3_access_key }} \
+              --set-string crunchy.pgBackRest.s3.secretKey=${{ secrets.s3_secret_key }} \
+              --values ${{ inputs.values }} postgres  .
+          else
+            helm upgrade --install --wait --values ${{ inputs.values }} postgres .
+          fi
+
+      - name: Add PR specific user to Crunchy DB # only for PRs
+        shell: bash
+        if: github.event_name == 'pull_request'
+        run: |
+            echo 'Adding PR specific user to Crunchy DB'
+            NEW_USER='{"databases":["app-${{ github.event.number }}"],"name":"app-${{ github.event.number }}"}'
+            CURRENT_USERS=$(oc get PostgresCluster/postgres-crunchy -o json | jq '.spec.users')
+            echo "${CURRENT_USERS}"
+
+            # check if current_users already contains the new_user
+            if echo "${CURRENT_USERS}" | jq -e ".[] | select(.name == \"app-${{ github.event.number }}\")" > /dev/null; then
+              echo "User already exists"
+              exit 0
+            fi
+
+            UPDATED_USERS=$(echo "$CURRENT_USERS" | jq --argjson NEW_USER "$NEW_USER" '. + [$NEW_USER]')
+            echo "$UPDATED_USERS"
+            PATCH_JSON=$(jq -n --argjson users "$UPDATED_USERS" '{"spec": {"users": $users}}')
+            echo "$PATCH_JSON"
+            oc patch PostgresCluster/postgres-crunchy --type=merge -p "${PATCH_JSON}"
+
+            # wait for sometime as it takes time to create the user, query the secret and check if it is created, otherwise wait in a loop for 10 rounds
+            for i in {1..10}; do
+              if oc get secret postgres-crunchy-pguser-app-${{ github.event.number }} -o jsonpath='{.metadata.name}' > /dev/null; then
+                echo "Secret created"
+                break
+              else
+                echo "Secret not created, waiting for 60 seconds"
+                sleep 60
+              fi
+            done
+
+            # Add public schema and grant to PR user
+            # get primary crunchy pod and remove the role and db
+            CRUNCHY_PG_PRIMARY_POD_NAME=$(oc get pods -l postgres-operator.crunchydata.com/role=master -o json | jq -r '.items[0].metadata.name')
+            echo "${CRUNCHY_PG_PRIMARY_POD_NAME}"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "CREATE SCHEMA IF NOT EXISTS public;"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON SCHEMA public TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO \"app-${{ github.event.number }}\";"
+            # TODO: remove these
+
+
+

.github/workflows/pr-close.yml

@@ -42,3 +42,50 @@ jobs:
           OC_SERVER: ${{ vars.OC_SERVER }}
           OC_TOKEN: ${{ secrets.OC_TOKEN }}
           PR_NUMBER: ${{ github.event.number }}
+
+  cleanup-crunchy:
+    name: Cleanup Ephemeral Crunchy Data
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      # OC setup
+      - uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4"
+
+      # OC Login
+      - run: |
+          # OC Login
+          oc login --token=${{ secrets.OC_TOKEN }} --server=${{ vars.OC_SERVER }}
+          oc project ${{ secrets.OC_NAMESPACE }} # Safeguard!
+      - run: |
+          # check if postgres-crunchy exists or else exit
+          oc get PostgresCluster/postgres-crunchy || exit 0
+          # Remove the user from the crunchy cluster yaml and apply the changes
+          USER_TO_REMOVE='{"databases":["app-${{ github.event.number }}"],"name":"app-${{ github.event.number }}"}'
+
+          echo 'getting current users from crunchy'
+          CURRENT_USERS=$(oc get PostgresCluster/postgres-crunchy -o json | jq '.spec.users')
+          echo "${CURRENT_USERS}"
+
+          # Remove the user from the list, 
+          UPDATED_USERS=$(echo "$CURRENT_USERS" | jq --argjson user "$USER_TO_REMOVE" 'map(select(. != $user))')
+
+          PATCH_JSON=$(jq -n --argjson users "$UPDATED_USERS" '{"spec": {"users": $users}}')
+          oc patch PostgresCluster/postgres-crunchy --type=merge -p "$PATCH_JSON"
+
+          # get primary crunchy pod and remove the role and db
+          CRUNCHY_PG_PRIMARY_POD_NAME=$(oc get pods -l postgres-operator.crunchydata.com/role=master -o json | jq -r '.items[0].metadata.name')
+
+          echo "${CRUNCHY_PG_PRIMARY_POD_NAME}"
+          # Terminate all connections to the database before trying terminate
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'app-${{ github.event.number }}' AND pid <> pg_backend_pid();"
+
+          # Drop the database and role
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "DROP DATABASE \"app-${{ github.event.number }}\" --cascade" 
+
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "DROP ROLE \"app-${{ github.event.number }}\" --cascade" 
+
+          echo "Database and Role for PR is cleaned."
+
+          exit 0

.github/workflows/pr-open.yml

@@ -32,16 +32,28 @@ jobs:
           tag_fallback: latest
           triggers: ('${{ matrix.package }}/')
 
+  # https://github.com/bcgov/quickstart-openshift
+  crunchy:
+    name: Deploy Crunchy
+    needs: [builds]
+    uses: ./.github/workflows/.dbdeployer.yml
+    with:
+      s3_enabled: false # no backups in dev
+    secrets:
+      oc_namespace: ${{ secrets.OC_NAMESPACE }}
+      oc_token: ${{ secrets.OC_TOKEN }}
+
   # https://github.com/bcgov/quickstart-openshift-helpers
   deploys:
     name: Deploys
-    needs: [builds]
+    needs: [builds, crunchy]
     uses: bcgov/quickstart-openshift-helpers/.github/workflows/[email protected]
     secrets:
       oc_namespace: ${{ secrets.OC_NAMESPACE }}
       oc_token: ${{ secrets.OC_TOKEN }}
     with:
-      triggers: ('backend/' 'frontend/' 'webeoc/' 'migrations/')
+      params: --set global.useCrunchy=true
+
 
   healthcheck:
     name: Healthcheck Deployment

charts/app/templates/backend/templates/deployment.yaml

@@ -31,9 +31,11 @@ spec:
         - name: {{ include "backend.fullname" . }}-init
           image: "{{ .Values.global.registry }}/{{ .Values.global.repository }}/migrations:{{ .Values.global.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ default "Always" .Values.backend.imagePullPolicy }}
+          {{- if not .Values.global.useCrunchy }}
           envFrom:
             - secretRef:
                 name: {{ .Release.Name }}-flyway
+          {{- end }}
           env:
             - name: NODE_TLS_REJECT_UNAUTHORIZED
               value: "0"
@@ -45,6 +47,13 @@ spec:
               value: "10"
             - name: FLYWAY_LOCATIONS
               value: "{{- if eq .Release.Namespace "c1c7ed-dev" -}}{{ .Values.global.secrets.flywayLocations.dev }}{{- else if eq .Release.Namespace "c1c7ed-test" -}}{{ .Values.global.secrets.flywayLocations.test }}{{- else if eq .Release.Namespace "c1c7ed-prod" -}}{{ .Values.global.secrets.flywayLocations.prod }}{{- else -}}filesystem:./flyway/sql{{- end }}"
+            {{- if .Values.global.useCrunchy }}
+            - name: FLYWAY_URL
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-crunchy-pguser-app-{{ .Values.global.tag | trimAll "\"" }}
+                  key: pgbouncer-jdbc-uri
+            {{- end }}
           resources:
             limits:
               cpu: 200m
@@ -68,6 +77,28 @@ spec:
               value: info
             - name: NODE_TLS_REJECT_UNAUTHORIZED
               value: "0"
+            {{- if .Values.global.useCrunchy }}
+            - name: POSTGRESQL_DATABASE
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-crunchy-pguser-app-{{ .Values.global.tag | trimAll "\"" }}
+                  key: dbname
+            - name: POSTGRESQL_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-crunchy-pguser-app-{{ .Values.global.tag | trimAll "\"" }}
+                  key: pgbouncer-host
+            - name: POSTGRESQL_USER
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-crunchy-pguser-app-{{ .Values.global.tag | trimAll "\"" }}
+                  key: user
+            - name: POSTGRESQL_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-crunchy-pguser-app-{{ .Values.global.tag | trimAll "\"" }}
+                  key: password
+            {{- end }}
           ports:
             - name: http
               containerPort: {{ .Values.backend.service.targetPort }}

charts/app/values.yaml

@@ -68,6 +68,8 @@ global:
   domain: "apps.silver.devops.gov.bc.ca" # it is apps.gold.devops.gov.bc.ca for gold cluster
   #-- the database Alias gives a nice way to switch to different databases, crunchy, patroni ... etc.
   databaseAlias: bitnami-pg
+  #-- use crunchy for the database, it is optional
+  useCrunchy: false
 
 #-- the components of the application, backend.
 backend:

charts/crunchy/Chart.yaml

@@ -0,0 +1,26 @@
+apiVersion: v2
+name: crunchy
+description: A Helm chart for Kubernetes deployment.
+icon: https://www.nicepng.com/png/detail/521-5211827_bc-icon-british-columbia-government-logo.png
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 5.5.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "5.5.1"
+