feat: Mongo DB backups proper implementation (#52)

* fix: update variables names and jumplines * fix: duplicate key * fix: include custom_storage_endpoint in aws block * fix: newlines control * fix: args for command * fix: environment azure variables * fix: include bucket path in all options * fix: delete databases variable * fix: update .sh * fix: update default shipyard-utils image
eduNEXT · Nov 23, 2023 · 16d27ea · 16d27ea
1 parent 5320b7f
commit 16d27ea
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 36 deletions.
diff --git a/drydock_backups/Dockerfile b/drydock_backups/Dockerfile
@@ -2,9 +2,9 @@ FROM mysql:5.7-debian as base
 
 RUN apt-get update && apt-get install -y wget gnupg2
 
-RUN wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | apt-key add -
+RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -
 
-RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/6.0 main" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list
+RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list
 
 RUN apt-get update && apt-get install -y \
     mongodb-org \

diff --git a/drydock_backups/README.rst b/drydock_backups/README.rst
@@ -1,4 +1,4 @@
-Drydock-backuyps
+Drydock-backups
 ================
 
 This is a tutor plugin used to easily store backups of mysql and mongodb databases through k8s cronjobs. It backups the databases and stores them in a bucket.

diff --git a/drydock_backups/docker-entrypoint.sh b/drydock_backups/docker-entrypoint.sh
@@ -8,12 +8,15 @@ MONGODB_URI="mongodb://"
 MONGODB_PORT=${MONGODB_PORT:-27017}
 MONGODB_HOST=${MONGODB_HOST:-mongodb}
 MONGODB_AUTHDB=${MONGODB_AUTHDB:-admin}
+MONGODB_AUTHDB_STRING="authSource=${MONGODB_AUTHDB}"
 
-if [ -n "${MONGODB_USERNAME}" ] && [ -n "${MONGODB_PASSWORD}" ]; then
+if [ -z ${MONGODB_USERNAME} ] && [ -z ${MONGODB_PASSWORD} ]; then
+    MONGODB_AUTHDB_STRING=""
+else
     MONGODB_URI+="${MONGODB_USERNAME}:${MONGODB_PASSWORD}@"
 fi
 
-MONGODB_URI+="${MONGODB_HOST}:${MONGODB_PORT}/${MONGODB_DATABASES}?authSource=${MONGODB_AUTHDB}"
+MONGODB_URI+="${MONGODB_HOST}:${MONGODB_PORT}/?${MONGODB_AUTHDB_STRING}"
 
 if [ "$1" = 'mysql' ]; then
       mysqldump -u $MYSQL_ROOT_USERNAME -h $MYSQL_HOST -P $MYSQL_PORT --password=$MYSQL_ROOT_PASSWORD \
@@ -32,7 +35,7 @@ if [ "$BACKUP_STORAGE_SERVICE" = 'aws-s3' ]; then
             aws --endpoint-url $BACKUP_CUSTOM_STORAGE_ENDPOINT s3 mv $FILENAME s3://$S3_BUCKET_NAME/$BUCKET_PATH/$1/
       fi
 elif [ "$BACKUP_STORAGE_SERVICE" = 'azure-blob' ]; then
-      azcopy cp $FILENAME "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_STORAGE_CONTAINER_NAME}/$BUCKET_PATH/${1}/$FILENAME?${AZURE_STORAGE_SAS_TOKEN}"
+      azcopy cp $FILENAME https://$AZURE_ACCOUNT_NAME.blob.core.windows.net/$AZURE_CONTAINER_NAME/$BUCKET_PATH/${1}/$FILENAME?$AZURE_CONTAINER_SAS_TOKEN
 else
     echo "Unknown storage service"
     exit 1

diff --git a/drydock_backups/patches/drydock-multipurpose-jobs b/drydock_backups/patches/drydock-multipurpose-jobs
@@ -33,35 +33,37 @@ spec:
                   value: '{{ BACKUP_MYSQL_USERNAME }}'
                 - name: MYSQL_ROOT_PASSWORD
                   value: '{{ BACKUP_MYSQL_PASSWORD }}'
-                {% if BACKUP_STORAGE_SERVICE == "aws-s3" %}
+                - name: BACKUP_STORAGE_SERVICE
+                  value: '{{ BACKUP_STORAGE_SERVICE }}'
+                - name: BUCKET_PATH
+                  value: '{{ BACKUP_BUCKET_PATH }}'
+                {%- if BACKUP_STORAGE_SERVICE == "aws-s3" %}
                 - name: AWS_ACCESS_KEY_ID
                   value: '{{ BACKUP_AWS_ACCESS_KEY }}'
                 - name: AWS_SECRET_ACCESS_KEY
                   value: '{{ BACKUP_AWS_SECRET_KEY }}'
                 - name: S3_BUCKET_NAME
                   value: '{{ BACKUP_BUCKET_NAME }}'
-                - name: BUCKET_PATH
-                  value: '{{ BACKUP_BUCKET_PATH }}'
-                {% elif BACKUP_STORAGE_SERVICE == "azure-blob" %}
+                  {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %}
+                - name: BACKUP_CUSTOM_STORAGE_ENDPOINT
+                  value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}'
+                  {%- endif %}
+                {%- elif BACKUP_STORAGE_SERVICE == "azure-blob" %}
                 - name: AZURE_CONTAINER_NAME
                   value: '{{ BACKUP_AZURE_CONTAINER_NAME }}'
                 - name: AZURE_CONTAINER_SAS_TOKEN
                   value: '{{ BACKUP_AZURE_CONTAINER_SAS_TOKEN }}'
                 - name: AZURE_ACCOUNT_NAME
                   value: '{{ BACKUP_AZURE_ACCOUNT_NAME }}'
-                {% endif %}
-                {% if BACKUP_CUSTOM_STORAGE_ENDPOINT %}
-                - name: BACKUP_CUSTOM_STORAGE_ENDPOINT
-                  value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}'
-                {% endif %}
-              {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
+                {%- endif %}
+              {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
               volumeMounts:
                 - mountPath: /data/
                   name: backup-volume
-              {% endif %}
+              {%- endif %}
               args:
                 - mysql
-          {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
+          {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
           volumes:
             - name: backup-volume
               ephemeral:
@@ -72,7 +74,7 @@ spec:
                     resources:
                       requests:
                         storage: {{ BACKUP_K8S_EPHEMERAL_VOLUME_SIZE }}
-          {% endif %}
+          {%- endif %}
 ---
 apiVersion: batch/v1
 kind: CronJob
@@ -108,37 +110,37 @@ spec:
                   value: '{{ BACKUP_MONGO_USERNAME }}'
                 - name: MONGODB_PASSWORD
                   value: '{{ BACKUP_MONGO_PASSWORD }}'
-                - name: MONGODB_DATABASES
-                  value: '{{ BACKUP_MONGODB_DATABASE }}'
-                {% if BACKUP_STORAGE_SERVICE == "aws-s3" %}
+                - name: BACKUP_STORAGE_SERVICE
+                  value: '{{ BACKUP_STORAGE_SERVICE }}'
+                - name: BUCKET_PATH
+                  value: '{{ BACKUP_BUCKET_PATH }}'
+                {%- if BACKUP_STORAGE_SERVICE == "aws-s3" %}
                 - name: AWS_ACCESS_KEY_ID
                   value: '{{ BACKUP_AWS_ACCESS_KEY }}'
                 - name: AWS_SECRET_ACCESS_KEY
                   value: '{{ BACKUP_AWS_SECRET_KEY }}'
                 - name: S3_BUCKET_NAME
                   value: '{{ BACKUP_BUCKET_NAME }}'
-                - name: BUCKET_PATH
-                  value: '{{ BACKUP_BUCKET_PATH }}'
-                  {% if BACKUP_CUSTOM_STORAGE_ENDPOINT %}
+                  {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %}
                 - name: BACKUP_CUSTOM_STORAGE_ENDPOINT
                   value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}'
-                  {% endif %}
-                {% elif BACKUP_STORAGE_SERVICE == "azure-blob" %}
+                  {%- endif %}
+                {%- elif BACKUP_STORAGE_SERVICE == "azure-blob" %}
                 - name: AZURE_CONTAINER_NAME
                   value: '{{ BACKUP_AZURE_CONTAINER_NAME }}'
                 - name: AZURE_CONTAINER_SAS_TOKEN
                   value: '{{ BACKUP_AZURE_CONTAINER_SAS_TOKEN }}'
                 - name: AZURE_ACCOUNT_NAME
                   value: '{{ BACKUP_AZURE_ACCOUNT_NAME }}'
-                {% endif %}
-              {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
+                {%- endif %}
+              {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
               volumeMounts:
                 - mountPath: /data/
                   name: backup-volume
-              {% endif %}
+              {%- endif %}
               args:
                 - mongo
-          {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
+          {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
           volumes:
             - name: backup-volume
               ephemeral:
@@ -149,4 +151,4 @@ spec:
                     resources:
                       requests:
                         storage: {{ BACKUP_K8S_EPHEMERAL_VOLUME_SIZE }}
-          {% endif %}
+          {%- endif %}
diff --git a/drydock_backups/patches/k8s-jobs b/drydock_backups/patches/k8s-jobs
diff --git a/drydock_backups/plugin.py b/drydock_backups/plugin.py
@@ -13,9 +13,9 @@
 config = {
     "defaults": {
         "VERSION": __version__,
-        "IMAGE": "ednxops/shipyard-utils:v1.1.0",
+        "IMAGE": "ednxops/shipyard-utils:v15.4.0",
         "CRON_SCHEDULE": '0 2 * * *',
-        "BACKUP_STORAGE_SERVICE": "aws-s3",
+        "STORAGE_SERVICE": "aws-s3",
         "AWS_ACCESS_KEY": "",
         "AWS_SECRET_KEY": "",
         "BUCKET_NAME": "",
@@ -30,7 +30,6 @@
         "MYSQL_PASSWORD": '{{ MYSQL_ROOT_PASSWORD }}',
         "MONGO_PASSWORD": '{{ MONGODB_PASSWORD }}',
         "MONGO_USERNAME": '{{ MONGODB_USERNAME }}',
-        "MONGODB_DATABASE": ""
     },
 }