added database seeding code and helpers

backend/Dockerfile

@@ -6,7 +6,8 @@ RUN  apt-get update \
   && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu1804-arm64-100.9.0.deb && \
     apt-get -y install ./mongodb-database-tools-*100.9.0.deb && \
-    rm -f mongodb-database-tools-*.deb
+    rm -f mongodb-database-tools-*.deb && \ 
+    apt install -y curl
 COPY package.json .
 RUN npm install --prefer-offline --no-audit
 COPY . .

backend/package.json

@@ -13,7 +13,8 @@
     "lint": "tsc --noEmit && eslint --ext .ts,.tsx .",
     "backup:mongodb": "node ./build/scripts/backup-mongodb.js",
     "download:mongodb": "node ./build/scripts/download-mongodb.js",
-    "update:catalog": "node ./build/scripts/update-catalog.js"
+    "update:catalog": "node ./build/scripts/update-catalog.js",
+    "seed:mongodb": "node ./build/scripts/seed-database.js"
   },
   "repository": {
     "type": "git",

backend/src/scripts/seed-database.ts

@@ -0,0 +1,144 @@
+import * as child from 'child_process';
+import mongooseLoader from '../bootstrap/loaders/mongoose';
+import { SemesterType, SemesterModel } from '../models/semester';
+import { dateToTerm } from '../utils/term';
+import { config } from '../config';
+
+const COLLECTION_RENAME_MAP:any = {
+    "sis_class": "class",
+    "sis_course": "course",
+    "sis_class_section": "sections"
+}
+
+const termOrder = ["Spring", "Summer", "Fall"]
+
+const FIRST_SEMESTER = {term: "Spring", year: 2012, active: false}
+
+const today = new Date();
+let current_semester = {term: dateToTerm(today), year: today.getFullYear(), active: true}
+
+const retrieveSeed = async () => {
+
+    const curl = child.spawn("curl", ["-O", "https://storage.googleapis.com/berkeleytime/public/mdb.archive"])
+    await new Promise((resolve) => {
+        curl.stdout.on("data", (data) => {
+            console.log(data.toString())
+        })
+        curl.on("close", resolve)
+    })
+
+    const restore = child.spawn("mongorestore", ["--uri", config.mongoDB.uri, "--drop", "--gzip", "--archive=mdb.archive"])
+    await new Promise((resolve) => {
+        restore.stderr.on("data", (data) => {
+            console.log(data.toString())
+        })
+        restore.on("close", resolve)
+    })
+
+}
+
+const fixSISCollections = async () => {
+    for (var original in COLLECTION_RENAME_MAP) {
+        const newName = COLLECTION_RENAME_MAP[original]
+        const dump = child.spawn("mongodump", ["--uri", config.mongoDB.uri, "-d", "bt", "-c", original])
+        await new Promise((resolve) => {
+            dump.stderr.on("data", (data) => {
+                console.log(data.toString())
+            })
+            dump.on("close", resolve)
+        })
+        const restore = child.spawn("mongorestore", ["--uri", config.mongoDB.uri, "--drop", "-d", "bt", "-c", newName, `dump/bt/${original}.bson`])
+        await new Promise((resolve) => {
+            restore.stderr.on("data", (data) => {
+                console.log(data.toString())
+            })
+            restore.on("close", resolve)
+        })
+    }
+}
+
+const nextSemester = (currentSemester : SemesterType) : SemesterType => {
+    const termIndex = termOrder.indexOf(currentSemester.term)
+    const finalTerm = (termIndex == termOrder.length - 1)
+    return {
+        term: (finalTerm) ? termOrder[0] : termOrder[termIndex + 1],
+        year: (finalTerm) ? currentSemester.year + 1 : currentSemester.year,
+        active: false
+    }
+}
+
+const createSemesters = async (firstSemester : SemesterType, currentSemester : SemesterType) => {
+    await SemesterModel.deleteMany({})
+    const semesters = []
+    for (let semester = firstSemester; 
+            semester.year != currentSemester.year || semester.term != currentSemester.term;
+                semester = nextSemester(semester)) {
+        semesters.push(semester)
+    }
+    semesters.push(currentSemester)
+    await SemesterModel.insertMany(semesters)
+    const numLoaded = await SemesterModel.count({})
+    if (numLoaded != semesters.length) {
+        throw Error(`Error while loading semesters: Got ${numLoaded}, expected ${semesters.length}`)
+    }
+    console.log(`Total number of semesters: ${numLoaded}`)
+}
+
+const runUpdateCatalog = async () => {
+    const process = child.spawn("npm", ["run", "update:catalog"])
+    await new Promise((resolve) => {
+        process.stdout.on("data", (data) => {
+            console.log(data.toString())
+        })
+        process.on("close", resolve)
+    })
+}
+
+
+(async () => {
+
+    // pre-reqs: Make sure .env is correctly setup
+
+    try {
+
+        await mongooseLoader()
+
+        console.log("\n===== RETRIEVING MONGODUMP ARCHIVE =====")
+        //await retrieveSeed()
+
+        console.log("\n===== COPY SIS_* COLLECTION DATA =====")
+        await fixSISCollections()
+
+        // allow arguments --term=TERM and --year=YEAR to set current semester
+        process.argv.forEach((arg) => {
+            if (arg.substring(0, 2) == "--") {
+                arg = arg.substring(2)
+                const equalSign = arg.indexOf("=")
+                if (equalSign == -1) throw new Error("Script arguments incorrectly formatted")
+                const key = arg.substring(0, equalSign)
+                const val = arg.substring(equalSign + 1)
+                switch (key) {
+                    case "term":
+                        current_semester.term = val
+                    case "year":
+                        current_semester.year = +val
+                }
+            }
+        })
+
+        console.log("\n===== CREATING SEMESTERS =====")
+        await createSemesters(
+            FIRST_SEMESTER,
+            current_semester
+        )
+
+        console.log("\n===== RUN UPDATE CATALOG SCRIPT =====")
+        await runUpdateCatalog()
+
+    } catch (err) {
+        console.error(err)
+        process.exit(1)
+    }
+
+    process.exit(0)
+})()

backend/src/scripts/update-catalog.ts

@@ -8,6 +8,7 @@ import { SISResponse } from '../utils/sis';
 import { MongooseBulkWriteOptions } from 'mongoose';
 import { ClassModel, ClassType } from '../models/class';
 import { SectionModel, SectionType } from '../models/section';
+import { performance } from 'perf_hooks';
 
 const SIS_COURSE_URL = 'https://gateway.api.berkeley.edu/sis/v4/courses';
 const SIS_CLASS_URL = 'https://gateway.api.berkeley.edu/sis/v1/classes';
@@ -164,6 +165,9 @@ const updateSections = async () => {
 
 (async () => {
     try {
+
+        const startTime = performance.now()
+
         await mongooseLoader();
 
         console.log("\n=== UPDATE COURSES ===")
@@ -174,6 +178,11 @@ const updateSections = async () => {
 
         console.log("\n=== UPDATE SECTIONS ===")
         await updateSections();
+
+        const endTime = performance.now()
+        const seconds = Math.round(endTime - startTime) / 1000
+        console.log(`Script took ${seconds}s to complete`)
+
     } catch (err) {
         console.error(err);
         process.exit(1);

backend/src/utils/term.ts

@@ -25,3 +25,14 @@ export function getTermStartMonth(term: TermInput) {
 
     return startDates[term.semester];
 }
+
+/**
+ * Gets the approximate term a given input date falls in
+ * Used for seed database script
+ */
+export function dateToTerm(date: Date) {
+    const month = date.getMonth()
+    if (month < 5) return "Spring"
+    if (month < 7) return "Summer"
+    return "Fall"
+}

docker-compose.yml

@@ -58,19 +58,19 @@ services:
     networks:
       - bt   
   # LOCAL S3 STORAGE - FOR TESTING MONGODB BACKUP SCRIPT
-  minio:
-    image: 'minio/minio:latest'
-    ports:
-      - '9000:9000'
-      - '9090:9090'
-    environment:
-      MINIO_ROOT_USER: 'root'
-      MINIO_ROOT_PASSWORD: 'password'
-    networks:
-      - bt
-    volumes:
-      - 'minio:/data/minio'
-    command: minio server /data/minio --console-address ":9090"
+  # minio:
+  #   image: 'minio/minio:latest'
+  #   ports:
+  #     - '9000:9000'
+  #     - '9090:9090'
+  #   environment:
+  #     MINIO_ROOT_USER: 'root'
+  #     MINIO_ROOT_PASSWORD: 'password'
+  #   networks:
+  #     - bt
+  #   volumes:
+  #     - 'minio:/data/minio'
+  #   command: minio server /data/minio --console-address ":9090"
 volumes:
   minio:
     driver: local