From bf33474f53564810aa4bb426081b3fcafcc0972a Mon Sep 17 00:00:00 2001
From: Norbert Gruszka <40759761+norbertgruszka@users.noreply.github.com>
Date: Fri, 10 Jan 2025 08:51:07 +0100
Subject: [PATCH] feat(): Add support for s3 storage in Kubernetes etcd backup
 job (#25)

* feat(): add S3 Environment variables

* feat(): create backup and push to s3

* fix(): user permssion

* fix(): use double quotes

* fix(): use double quotes

* feat(): update CA Certificates and fix typos
---
 Dockerfile         |   9 ++++
 README.md          |   8 ++++
 backup-config.yaml |   6 +++
 backup.sh          | 113 ++++++++++++++++++++++++++++-----------------
 4 files changed, 93 insertions(+), 43 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0aea52c..9863f1a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,6 +10,11 @@ RUN microdnf update -y && rm -rf /var/cache/yum
 # hadolint ignore=DL3041
 RUN microdnf install -y curl findutils gzip tar \
     && microdnf clean all
+
+RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc.rpm \
+    && rpm -ih mc.rpm \
+    && rm mc.rpm
+    
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN /bin/bash -o pipefail -c "\
     curl -L https://github.com/etcd-io/etcd/releases/download/v3.5.15/etcd-v3.5.15-linux-amd64.tar.gz \
@@ -19,4 +24,8 @@ RUN /bin/bash -o pipefail -c "\
     && mv /tmp/etcd /usr/local/bin/ \
     "
 
+ENV MC_CONFIG_DIR=/opt/mc/config
+RUN mkdir -p $MC_CONFIG_DIR && \
+    chown 1000:1000 $MC_CONFIG_DIR
+
 CMD ["/usr/local/bin/backup.sh"]
diff --git a/README.md b/README.md
index ac0cf3c..1baee22 100644
--- a/README.md
+++ b/README.md
@@ -82,6 +82,12 @@ kubectl edit -n etcd-backup cm/backup-config
 ```
 
 The following options are used:
+- `ETCD_BACKUP_S3`: Use S3 to store etcd-backup snapshots
+- `ETCD_BACKUP_S3_NAME`: MinIO client host alias name
+- `ETCD_BACKUP_S3_HOST`: S3 host endpoint (with scheme)
+- `ETCD_BACKUP_S3_BUCKET`: S3 bucket name
+- `ETCD_BACKUP_S3_ACCESS_KEY`: access key to access S3 bucket
+- `ETCD_BACKUP_S3_SECRET_KEY`: secret key to access S3 bucket
 - `ETCD_BACKUP_SUBDIR`: Sub directory on PVC that should be used to store the backup. If it does not exist it will be created.
 - `ETCD_BACKUP_DIRNAME`: Directory name for a single backup. This is a format string used by
 [`date`](https://man7.org/linux/man-pages/man1/date.1.html)
@@ -94,6 +100,8 @@ The following options are used:
 - `ETCD_BACKUP_UMASK`: Umask used inside the script to set restrictive permission on written files, as they contain sensitive information.
 - `ENDPOINT`: The IP address of the etcd endpoint, without scheme or port, e.g. `"192.168.39.86"`.
 
+Note that the storage type is exclusive. This means it is either S3 or PVC. In case of using S3 we do not manage the retention within the backup script. We suggest using a rentention policy on the S3 bucket itself. This can be done thanks to an objects expiration configuration as described in the object lifecycle management [documentation](https://min.io/docs/minio/linux/administration/object-management/object-lifecycle-management.html#object-expiration).
+
 Changing the schedule be done in the CronJob directly, with `spec.schedule`:
 ```
 kubectl edit -n etcd-backup cronjob/etcd-backup
diff --git a/backup-config.yaml b/backup-config.yaml
index 47788f1..d47f352 100644
--- a/backup-config.yaml
+++ b/backup-config.yaml
@@ -4,6 +4,12 @@ metadata:
   name: backup-config
   namespace: etcd-backup
 data:
+  ETCD_BACKUP_S3: "false"
+  ETCD_BACKUP_S3_NAME: "minio"
+  ETCD_BACKUP_S3_HOST: "http://minio.local:9000"
+  ETCD_BACKUP_S3_BUCKET: "etcd-backup"
+  ETCD_BACKUP_S3_ACCESS_KEY: "randomaccesskey"
+  ETCD_BACKUP_S3_SECRET_KEY: "secretkey"
   ETCD_BACKUP_SUBDIR: "/"
   ETCD_BACKUP_DIRNAME: "+etcd-backup-%FT%T%:z"
   ETCD_BACKUP_EXPIRE_TYPE: "days"
diff --git a/backup.sh b/backup.sh
index d15c7c0..196a5b8 100755
--- a/backup.sh
+++ b/backup.sh
@@ -3,7 +3,7 @@
 # backup.sh General etcd backup script
 ################################################################################
 #
-# Copyright (C) 2023 Adfinis AG
+# Copyright (C) 2024 Adfinis AG
 #                    https://adfinis.com
 #                    info@adfinis.com
 #
@@ -32,50 +32,77 @@
 
 set -xeuo pipefail
 
-# set proper umask
-umask "${ETCD_BACKUP_UMASK}"
+# check storage type
+if [ "${ETCD_BACKUP_S3}" = "true" ]; then
+    # prepare & push backup to S3
 
-# validate expire type
-case "${ETCD_BACKUP_EXPIRE_TYPE}" in
-    days|count|never) ;;
-    *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;;
-esac
+    # update CA trust
+    update-ca-trust
 
-# validate  expire numbers
-if [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "days" ]; then
-  case "${ETCD_BACKUP_KEEP_DAYS}" in
-    ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;;
-    *) ;;
-  esac
-elif [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "count" ]; then
-  case "${ETCD_BACKUP_KEEP_COUNT}" in
-    ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;;
-    *) ;;
+    # configure mcli assuming the bucket already exists
+    bash +o history
+    mcli alias set "${ETCD_BACKUP_S3_NAME}" "${ETCD_BACKUP_S3_HOST}" "${ETCD_BACKUP_S3_ACCESS_KEY}" "${ETCD_BACKUP_S3_SECRET_KEY}"
+    bash -o history
+
+    # make dirname
+    BACKUP_FOLDER="$( date "${ETCD_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; }
+
+    # make necessary directory
+    mkdir -p "/tmp/etcd-backup/${BACKUP_FOLDER}"
+
+    # create backup to temporary location
+    ETCDCTL_API=3 etcdctl --endpoints "${ENDPOINT}:2379" --cacert="/etc/kubernetes/pki/etcd-ca/ca.crt"  --cert="/etc/kubernetes/pki/etcd-peer/tls.crt" --key="/etc/kubernetes/pki/etcd-peer/tls.key" snapshot save "/tmp/etcd-backup/${BACKUP_FOLDER}/snapshot.db"
+    ETCDCTL_API=3 etcdutl --write-out=table snapshot status "/tmp/etcd-backup/${BACKUP_FOLDER}/snapshot.db"
+
+    # move files to S3 and delete temporary files
+    mcli mv -r /tmp/etcd-backup/* "${ETCD_BACKUP_S3_NAME}"/"${ETCD_BACKUP_S3_BUCKET}"
+    rm -rv /tmp/etcd-backup
+else
+  # set proper umask
+  umask "${ETCD_BACKUP_UMASK}"
+
+  # validate expire type
+  case "${ETCD_BACKUP_EXPIRE_TYPE}" in
+      days|count|never) ;;
+      *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;;
   esac
-fi
 
-# make dirname and cleanup paths
-BACKUP_FOLDER="$( date "${ETCD_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; }
-BACKUP_PATH="$( realpath -m "${ETCD_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )"
-BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )"
-BACKUP_ROOTPATH="$( realpath -m "/backup/${ETCD_BACKUP_SUBDIR}" )"
-
-# make nescesary directorys
-mkdir -p "/tmp/etcd-backup"
-mkdir -p "${BACKUP_PATH_POD}"
-
-# create backup to temporary location
-ETCDCTL_API=3 etcdctl --endpoints "${ENDPOINT}:2379" --cacert='/etc/kubernetes/pki/etcd-ca/ca.crt'  --cert='/etc/kubernetes/pki/etcd-peer/tls.crt' --key='/etc/kubernetes/pki/etcd-peer/tls.key' snapshot save /tmp/etcd-backup/snapshot.db
-ETCDCTL_API=3 etcdutl --write-out=table snapshot status /tmp/etcd-backup/snapshot.db
-
-# move files to pvc and delete temporary files
-mv /tmp/etcd-backup/* "${BACKUP_PATH_POD}"
-rm -rv /tmp/etcd-backup
-
-# expire backup
-if [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "days" ]; then
-  find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1  -type d -mtime "+${ETCD_BACKUP_KEEP_DAYS}" -exec rm -rv {} +
-elif [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "count" ]; then
-  # shellcheck disable=SC3040,SC2012
-  ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${ETCD_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}"
+  # validate  expire numbers
+  if [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "days" ]; then
+    case "${ETCD_BACKUP_KEEP_DAYS}" in
+      ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;;
+      *) ;;
+    esac
+  elif [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "count" ]; then
+    case "${ETCD_BACKUP_KEEP_COUNT}" in
+      ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;;
+      *) ;;
+    esac
+  fi
+
+  # make dirname and cleanup paths
+  BACKUP_FOLDER="$( date "${ETCD_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; }
+  BACKUP_PATH="$( realpath -m "${ETCD_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )"
+  BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )"
+  BACKUP_ROOTPATH="$( realpath -m "/backup/${ETCD_BACKUP_SUBDIR}" )"
+
+  # make nescesary directorys
+  mkdir -p "/tmp/etcd-backup"
+  mkdir -p "${BACKUP_PATH_POD}"
+
+  # create backup to temporary location
+  ETCDCTL_API=3 etcdctl --endpoints "${ENDPOINT}:2379" --cacert='/etc/kubernetes/pki/etcd-ca/ca.crt'  --cert='/etc/kubernetes/pki/etcd-peer/tls.crt' --key='/etc/kubernetes/pki/etcd-peer/tls.key' snapshot save /tmp/etcd-backup/snapshot.db
+  ETCDCTL_API=3 etcdutl --write-out=table snapshot status /tmp/etcd-backup/snapshot.db
+
+  # move files to pvc and delete temporary files
+  mv /tmp/etcd-backup/* "${BACKUP_PATH_POD}"
+  rm -rv /tmp/etcd-backup
+
+  # expire backup
+  if [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "days" ]; then
+    find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1  -type d -mtime "+${ETCD_BACKUP_KEEP_DAYS}" -exec rm -rv {} +
+  elif [ "${ETCD_BACKUP_EXPIRE_TYPE}" = "count" ]; then
+    # shellcheck disable=SC3040,SC2012
+    ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${ETCD_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}"
+  fi
 fi