DBC22-2085: Backup Media and improve log handling

bcgov · Apr 30, 2024 · c9ec45a · c9ec45a
1 parent 95025ca
commit c9ec45a
Show file tree

Hide file tree

Showing 35 changed files with 721 additions and 279 deletions.
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -106,27 +106,27 @@ jobs:
         username: ${{ env.REGISTRY_USER }}
         password: ${{ env.REGISTRY_PASSWORD }}
 
-  build-logprocessing:
+  build-openshiftjobs:
     runs-on: ubuntu-latest
-    name: Build & Push Docker Log Processing Image
+    name: Build & Push Docker OpenShift Jobs Image
     environment: 
       name: dev
 
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
 
-    - name: Build Log Processing
+    - name: Build OpenShift Jobs
       id: build_image
       uses: redhat-actions/buildah-build@v2
       with:
         context: .
         layers: true
-        image: drivebc-logprocessing
+        image: drivebc-openshiftjobs
         tags: latest latest-dev ${{ github.sha }}
         labels: |
           app=drivebc
-        containerfiles: ./compose/logprocessing/DockerFile
+        containerfiles: ./compose/openshiftjobs/DockerFile
 
     - name: Push to Github Packages
       uses: redhat-actions/push-to-registry@v2
@@ -138,7 +138,7 @@ jobs:
         password: ${{ env.REGISTRY_PASSWORD }}
 
   versionUpdate:
-    needs: [build-static, build-backend, build-redis, build-logprocessing]
+    needs: [build-static, build-backend, build-redis, build-openshiftjobs]
     runs-on: ubuntu-latest
     name: Deploy Latest Images
     environment:
@@ -159,4 +159,4 @@ jobs:
     - name: Helm upgrade on OpenShift Environment
       run: |
         helm dependency update ./infrastructure/main
-        helm upgrade dev-drivebc -f ./infrastructure/main/values-dev.yaml ./infrastructure/main --set django.image.tag="${{ github.sha }}" --set redis.image.tag="${{ github.sha }}" --set static.image.tag="${{ github.sha }}" --set tasks.image.tag="${{ github.sha }}" --set logprocessing.image.tag="${{ github.sha }}"
+        helm upgrade dev-drivebc -f ./infrastructure/main/values-dev.yaml ./infrastructure/main --set django.image.tag="${{ github.sha }}" --set redis.image.tag="${{ github.sha }}" --set static.image.tag="${{ github.sha }}" --set tasks.image.tag="${{ github.sha }}" --set openshiftjobs.image.tag="${{ github.sha }}"
diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml
@@ -59,11 +59,11 @@ jobs:
         tags: |
           latest-prod
 
-    - name: Add latest-uat tag to logprocessing image
+    - name: Add latest-uat tag to openshiftjobs image
       uses: shrink/actions-docker-registry-tag@v4
       with:
         registry: ghcr.io
-        repository: bcgov/drivebc-logprocessing
+        repository: bcgov/drivebc-openshiftjobs
         target: ${{ needs.deploy-pre.outputs.tag }}
         tags: |
           latest-prod
@@ -90,4 +90,4 @@ jobs:
     - name: Helm upgrade on OpenShift Environment
       run: |
         helm dependency update ./infrastructure/main
-        helm upgrade prod-drivebc -f ./infrastructure/main/values-prod.yaml ./infrastructure/main --set django.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set redis.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set static.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.deploy-pre.outputs.tag }}"
+        helm upgrade prod-drivebc -f ./infrastructure/main/values-prod.yaml ./infrastructure/main --set django.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set redis.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set static.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set openshiftjobs.image.tag="${{ needs.deploy-pre.outputs.tag }}"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -152,25 +152,25 @@ jobs:
         username: ${{ env.REGISTRY_USER }}
         password: ${{ env.REGISTRY_PASSWORD }}
 
-  build-logprocessing:
+  build-openshiftjobs:
     needs: [build-pre]
     runs-on: ubuntu-latest
-    name: Build & Push Docker Log-Processing Image
+    name: Build & Push Docker OpenShift Jobs Image
     environment: test
 
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
 
-    - name: Build Log-Processing
+    - name: Build OpenShift Jobs
       id: build_image
       uses: redhat-actions/buildah-build@v2
       with:
-        image: drivebc-logprocessing
+        image: drivebc-openshiftjobs
         tags: latest latest-test ${{ github.sha }} ${{ needs.build-pre.outputs.tag }}
         labels: |
           app=drivebc
-        containerfiles: ./compose/logprocessing/DockerFile
+        containerfiles: ./compose/openshiftjobs/DockerFile
 
     - name: Push to Github Packages
       uses: redhat-actions/push-to-registry@v2
@@ -182,7 +182,7 @@ jobs:
         password: ${{ env.REGISTRY_PASSWORD }}
 
   versionUpdate:
-    needs: [build-pre, build-static, build-backend, build-redis, build-logprocessing]
+    needs: [build-pre, build-static, build-backend, build-redis, build-openshiftjobs]
     runs-on: ubuntu-latest
     name: Deploy Latest Images
     environment:
@@ -205,4 +205,4 @@ jobs:
     - name: Helm upgrade on OpenShift Environment
       run: |
         helm dependency update ./infrastructure/main
-        helm upgrade test-drivebc -f ./infrastructure/main/values-test.yaml ./infrastructure/main --set django.image.tag="${{ needs.build-pre.outputs.tag }}" --set redis.image.tag="${{ needs.build-pre.outputs.tag }}" --set static.image.tag="${{ needs.build-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.build-pre.outputs.tag }}" --set logprocessing.image.tag="${{ needs.build-pre.outputs.tag }}"
+        helm upgrade test-drivebc -f ./infrastructure/main/values-test.yaml ./infrastructure/main --set django.image.tag="${{ needs.build-pre.outputs.tag }}" --set redis.image.tag="${{ needs.build-pre.outputs.tag }}" --set static.image.tag="${{ needs.build-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.build-pre.outputs.tag }}" --set openshiftjobs.image.tag="${{ needs.build-pre.outputs.tag }}"
diff --git a/.github/workflows/test_manual.yml b/.github/workflows/test_manual.yml
@@ -131,25 +131,25 @@ jobs:
         username: ${{ env.REGISTRY_USER }}
         password: ${{ env.REGISTRY_PASSWORD }}
 
-  build-logprocessing:
+  build-openshiftjobs:
     needs: [build-pre]
     runs-on: ubuntu-latest
-    name: Build & Push Docker Log-Processing Image
+    name: Build & Push Docker OpenShift Jobs Image
     environment: test
 
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
 
-    - name: Build Log-Processing
+    - name: Build OpenShift Jobs
       id: build_image
       uses: redhat-actions/buildah-build@v2
       with:
-        image: drivebc-logprocessing
+        image: drivebc-openshiftjobs
         tags: latest latest-test ${{ github.sha }} ${{ needs.build-pre.outputs.tag }}
         labels: |
           app=drivebc
-        containerfiles: ./compose/logprocessing/DockerFile
+        containerfiles: ./compose/openshiftjobs/DockerFile
 
     - name: Push to Github Packages
       uses: redhat-actions/push-to-registry@v2
@@ -161,7 +161,7 @@ jobs:
         password: ${{ env.REGISTRY_PASSWORD }}
 
   versionUpdate:
-    needs: [build-pre, build-static, build-backend, build-redis, build-logprocessing]
+    needs: [build-pre, build-static, build-backend, build-redis, build-openshiftjobs]
     runs-on: ubuntu-latest
     name: Deploy Latest Images
     environment:
@@ -184,4 +184,4 @@ jobs:
     - name: Helm upgrade on OpenShift Environment
       run: |
         helm dependency update ./infrastructure/main
-        helm upgrade test-drivebc -f ./infrastructure/main/values-test.yaml ./infrastructure/main --set django.image.tag="${{ needs.build-pre.outputs.tag }}" --set redis.image.tag="${{ needs.build-pre.outputs.tag }}" --set static.image.tag="${{ needs.build-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.build-pre.outputs.tag }}" --set logprocessing.image.tag="${{ needs.build-pre.outputs.tag }}"
+        helm upgrade test-drivebc -f ./infrastructure/main/values-test.yaml ./infrastructure/main --set django.image.tag="${{ needs.build-pre.outputs.tag }}" --set redis.image.tag="${{ needs.build-pre.outputs.tag }}" --set static.image.tag="${{ needs.build-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.build-pre.outputs.tag }}" --set openshiftjobs.image.tag="${{ needs.build-pre.outputs.tag }}"
diff --git a/.github/workflows/uat.yml b/.github/workflows/uat.yml
@@ -61,11 +61,11 @@ jobs:
         tags: |
           latest-uat
 
-    - name: Add latest-uat tag to logprocessing image
+    - name: Add latest-uat tag to openshiftjobs image
       uses: shrink/actions-docker-registry-tag@v4
       with:
         registry: ghcr.io
-        repository: bcgov/drivebc-logprocessing
+        repository: bcgov/drivebc-openshiftjobs
         target: ${{ needs.deploy-pre.outputs.tag }}
         tags: |
           latest-uat
@@ -92,4 +92,4 @@ jobs:
     - name: Helm upgrade on OpenShift Environment
       run: |
         helm dependency update ./infrastructure/main
-        helm upgrade uat-drivebc -f ./infrastructure/main/values-uat.yaml ./infrastructure/main --set django.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set redis.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set static.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set logprocessing.image.tag="${{ needs.deploy-pre.outputs.tag }}"
+        helm upgrade uat-drivebc -f ./infrastructure/main/values-uat.yaml ./infrastructure/main --set django.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set redis.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set static.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set openshiftjobs.image.tag="${{ needs.deploy-pre.outputs.tag }}"
diff --git a/compose/logprocessing/DockerFile b/compose/logprocessing/DockerFile
diff --git a/compose/logprocessing/entrypoint.sh b/compose/logprocessing/entrypoint.sh
diff --git a/compose/openshiftjobs/DockerFile b/compose/openshiftjobs/DockerFile
@@ -0,0 +1,23 @@
+FROM alpine:3
+RUN apk update && apk upgrade
+#Need goaccess 1.92 for a timezone fix. Once that version is in the regular branch, we can pull it from there.
+RUN apk add goaccess --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main
+
+RUN apk add --no-cache \
+    aws-cli \
+    bash \
+    coreutils \
+    tzdata
+
+COPY ./compose/openshiftjobs/entrypoint.sh /
+COPY ./compose/openshiftjobs/scripts/analyzeexportlogs.sh /scripts/
+COPY ./compose/openshiftjobs/scripts/backupmediapvc.sh /scripts/
+COPY ./compose/openshiftjobs/scripts/ziplogs.sh /scripts/
+
+RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh
+RUN sed -i 's/\r$//g' /scripts/analyzeexportlogs.sh && chmod +x /scripts/analyzeexportlogs.sh
+RUN sed -i 's/\r$//g' /scripts/backupmediapvc.sh && chmod +x /scripts/backupmediapvc.sh
+RUN sed -i 's/\r$//g' /scripts/ziplogs.sh && chmod +x /scripts/ziplogs.sh
+
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/compose/openshiftjobs/entrypoint.sh b/compose/openshiftjobs/entrypoint.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Check the number of arguments
+if [ "$#" -lt 1 ]; then
+    echo "Usage: $0 <script_name> [args...]"
+    exit 1
+fi
+
+# Determine which script to run based on the first argument
+case "$1" in
+    backupmediapvc)
+        # Run backupmediapvc.sh which will backup the data from the django-media pvc to s3 storage
+        /scripts/backupmediapvc.sh
+        ;;
+    ziplogs)
+        # Run ziplogs.sh which will zip all files that were created in the previous hour or older in the nginx log storage pvc
+        /scripts/ziplogs.sh
+        ;;
+    analyzeexportlogs)
+        # Run analyzeexportlogs with additional arguments which will send the specified days logs through goaccess and then upload to s3.
+        shift # Remove the first argument (script number)
+        /scripts/analyzeexportlogs.sh "$@"
+        ;;
+    *)
+        echo "Invalid script"
+        exit 1
+        ;;
+esac