added local minio s3 and backup:mongodb script

.env.template

@@ -9,3 +9,9 @@ REDIS_URI=redis://redis:6379
 GOOGLE_CLIENT_ID=_
 GOOGLE_CLIENT_SECRET=_
 SESSION_SECRET=_
+
+S3_ENDPOINT=minio
+S3_PORT=9000
+S3_ACCESS_KEY_ID=root
+S3_SECRET_ACCESS_KEY=password
+S3_MONGO_BACKUP_BUCKET=mongo-backup

backend/Dockerfile

@@ -1,6 +1,12 @@
 FROM node:16.14.2-slim AS builder
 RUN mkdir /backend
 WORKDIR /backend
+RUN  apt-get update \
+  && apt-get install -y wget \
+  && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu1804-arm64-100.9.0.deb && \
+    apt-get -y install ./mongodb-database-tools-*100.9.0.deb && \
+    rm -f mongodb-database-tools-*.deb
 COPY package.json .
 RUN npm install --prefer-offline --no-audit
 COPY . .

backend/package.json

@@ -10,7 +10,8 @@
     "test": "jest --watch",
     "coverage": "jest --coverage",
     "generate": "graphql-codegen --config codegen.ts",
-    "lint": "tsc --noEmit && eslint --ext .ts,.tsx ."
+    "lint": "tsc --noEmit && eslint --ext .ts,.tsx .",
+    "backup:mongodb": "node ./build/scripts/backup-mongodb.js"
   },
   "repository": {
     "type": "git",
@@ -35,7 +36,7 @@
     "@types/helmet": "0.0.46",
     "@types/jest": "^25.2.1",
     "@types/lodash": "^4.14.186",
-    "@types/node": "^13.13.5",
+    "@types/node": "^13.13.52",
     "@types/passport-google-oauth20": "^2.0.11",
     "@typescript-eslint/eslint-plugin": "^5.40.1",
     "@typescript-eslint/parser": "^5.40.1",
@@ -61,6 +62,7 @@
     "graphql-type-json": "^0.3.2",
     "helmet": "^3.22.0",
     "lodash": "^4.17.21",
+    "minio": "^7.1.3",
     "mongodb": "^4.10.0",
     "mongoose": "^6.8.0",
     "passport": "^0.6.0",

backend/src/config.ts

@@ -24,6 +24,11 @@ export interface Config {
   SESSION_SECRET: string;
   GOOGLE_CLIENT_ID: string;
   GOOGLE_CLIENT_SECRET: string;
+  S3_ENDPOINT: string,
+  S3_PORT: number,
+  S3_ACCESS_KEY_ID: string,
+  S3_SECRET_ACCESS_KEY: string,
+  S3_MONGO_BACKUP_BUCKET: string,
 }
 
 // All your secrets, keys go here
@@ -39,4 +44,9 @@ export const config: Config = {
   SESSION_SECRET: env("SESSION_SECRET"),
   GOOGLE_CLIENT_ID: env("GOOGLE_CLIENT_ID"),
   GOOGLE_CLIENT_SECRET: env("GOOGLE_CLIENT_SECRET"),
+  S3_ENDPOINT: env("S3_ENDPOINT"),
+  S3_PORT: parseInt(env("S3_PORT")),
+  S3_ACCESS_KEY_ID: env("S3_ACCESS_KEY_ID"),
+  S3_SECRET_ACCESS_KEY: env("S3_SECRET_ACCESS_KEY"),
+  S3_MONGO_BACKUP_BUCKET: env("S3_MONGO_BACKUP_BUCKET"),
 };

backend/src/scripts/backup-mongodb.ts

@@ -0,0 +1,91 @@
+import * as Minio from 'minio';
+import * as child from 'child_process';
+import * as Fs from "fs";
+import { config } from "../config";
+
+const ensureBucketExists = async (client : Minio.Client) => {
+    const exists = await client.bucketExists(config.S3_MONGO_BACKUP_BUCKET)
+    if (!exists) {
+        const res = await client.makeBucket(config.S3_MONGO_BACKUP_BUCKET)
+        console.log(`Bucket ${config.S3_MONGO_BACKUP_BUCKET} created`)
+    } 
+}
+
+const error = async (client : Minio.Client, toDelete : string[]) => {
+    client.removeObjects(config.S3_MONGO_BACKUP_BUCKET, toDelete, (e) => {
+        if (e) {
+            console.error("Unable to delete incorrect objects. Please do so manually.")
+        }
+    });
+}
+
+(async () => {
+
+    const client = new Minio.Client({
+        endPoint: config.S3_ENDPOINT,
+        port: config.S3_PORT,
+        useSSL: config.S3_ENDPOINT != "minio",
+        accessKey: config.S3_ACCESS_KEY_ID,
+        secretKey: config.S3_SECRET_ACCESS_KEY,
+    })
+
+    await ensureBucketExists(client)
+
+    const dump = child.spawn("mongodump", ["--uri", config.mongoDB.uri])
+
+    console.log("RUNNING MONGODUMP\n---------------------\n")
+
+    const dumpedFiles:string[][] = [];
+
+    const fileFinder = new RegExp(/(?:writing )(.*)(?: to )(.*)\n/)
+    dump.stderr.on('data', (data) => {
+        // mongodump output goes by default to stderr
+        const matches = data.toString().match(fileFinder)
+        if (matches != null) {
+            dumpedFiles.push([matches[1], matches[2]])
+        }
+        console.log(data.toString())
+    });
+
+    dump.on('close', (code) => {
+        if (code != 0) {
+            console.error('mongodump failed')
+            process.exit(1);
+        }
+        const complete:string[] = []
+        let puts = 0
+        const folderName = (new Date()).getTime().toString()
+        console.log(`--------------------------\nSAVING BACKUP TO: ${folderName} \n\n`)
+        let errored = false
+        dumpedFiles.forEach((file) => {
+            const fileStream = Fs.createReadStream(file[1])
+            Fs.stat(file[1], (err, stats) => {
+                if (err) {
+                    errored = true
+                    return
+                }
+                const name = `${folderName}/${file[0]}`
+                client.putObject(config.S3_MONGO_BACKUP_BUCKET, name, fileStream, stats.size, (err:any, objInfo:any) => {
+                    puts++
+                    if (err) {
+                        console.error(err) // err should be null
+                        errored = true
+                    } else {
+                        console.log('Object Created:', objInfo)
+                        complete.push(name)
+                    }
+                    if (puts == dumpedFiles.length) {
+                        if (errored) {
+                            error(client, complete)
+                            console.log("\n--------------------\nBACKUP FAILED")
+                        } else {
+                            console.log("\n--------------------\nBACKUP SUCCESSFUL")
+                        }
+                    }
+                })
+            })
+        })
+
+    }); 
+
+})();

docker-compose.yml

@@ -57,3 +57,20 @@ services:
     restart: always
     networks:
       - bt
+  minio:
+    image: 'minio/minio:latest'
+    ports:
+      - '9000:9000'
+      - '9090:9090'
+    environment:
+      MINIO_ROOT_USER: 'root'
+      MINIO_ROOT_PASSWORD: 'password'
+    networks:
+      - bt
+    volumes:
+      - 'minio:/data/minio'
+    command: minio server /data/minio --console-address ":9090"
+volumes:
+  minio:
+    driver: local
+