Added handlers to build and update databases. (#1)

These handlers accept functions for listing and fetching the files, which
abstracts away the origin of the files (local/S3) from the database actions.
The idea is that the handlers themselves are wrapped in command-line scripts
that decide what file source to use and pass along the relevant functions.

src/handlers/freshHandler.js

@@ -0,0 +1,51 @@
+import * as fs from "fs";
+import { addVersion } from "../sqlite/addVersion.js"; 
+import { createTables } from "../sqlite/createTables.js"; 
+import Database from "better-sqlite3"
+
+export async function freshHandler(db_paths, list_projects, list_assets, list_versions, find_latest, read_summary, read_metadata, tokenizable) {
+    const db_handles = {};
+    for (const [k, v] of Object.entries(db_paths)) {
+        if (fs.existsSync(v)) {
+            fs.unlinkSync(v); // remove any existing file.
+        }
+        const db = Database(v);
+        createTables(db);
+        db_handles[k] = db;
+    }
+
+    const all_projects = await list_projects();
+    for (const project of all_projects) {
+        await internal_freshProject(db_handles, project, list_assets, list_versions, find_latest, read_summary, read_metadata, tokenizable);
+    }
+}
+
+// Only exported for the purpose of re-use in manualHandler.js.
+export async function internal_freshProject(db_handles, project, list_assets, list_versions, find_latest, read_summary, read_metadata, tokenizable) {
+    const all_assets = await list_assets(project);
+    for (const asset of all_assets) {
+        await internal_freshAsset(db_handles, project, asset, list_versions, find_latest, read_summary, read_metadata, tokenizable);
+    }
+}
+
+export async function internal_freshAsset(db_handles, project, asset, list_versions, find_latest, read_summary, read_metadata, tokenizable) {
+    const latest = find_latest(project, asset);
+    if (latest == null) { // short-circuit if latest=null, as that means that there are no non-probational versions.
+        return; 
+    }
+    const all_versions = await list_versions(project, asset);
+    for (const version of all_versions) {
+        await internal_freshVersion(db_handles, project, asset, version, latest, read_summary, read_metadata, tokenizable);
+    }
+}
+
+export async function internal_freshVersion(db_handles, project, asset, version, latest, read_summary, read_metadata, tokenizable) {
+    const summ = await read_summary(project, asset, version);
+    if ("on_probation" in summ && summ.on_probation) {
+        return;
+    }
+    const output = await read_metadata(project, asset, version, Object.keys(db_handles));
+    for (const [e, db] of Object.entries(db_handles)) {
+        addVersion(db, project, asset, version, (latest == version), output[e], tokenizable);
+    }
+}

src/handlers/manualHandler.js

@@ -0,0 +1,34 @@
+import { deleteVersion } from "../sqlite/deleteVersion.js"; 
+import { deleteAsset } from "../sqlite/deleteAsset.js"; 
+import { deleteProject } from "../sqlite/deleteProject.js"; 
+import * as fresh from "./freshHandler.js";
+import Database from "better-sqlite3"
+
+export async function manualHandler(db_paths, project, asset, version, list_assets, list_versions, find_latest, read_summary, read_metadata, tokenizable) {
+    const db_handles = {};
+    for (const [k, v] of Object.entries(db_paths)) {
+        db_handles[k] = Database(v);
+    }
+
+    if (asset == null && version == null) {
+        for (const db of Object.values(db_handles)) {
+            deleteProject(db, project);
+        }
+        await fresh.internal_freshProject(db_handles, project, list_assets, list_versions, find_latest, read_summary, read_metadata, tokenizable);
+
+    } else if (version == null) {
+        for (const db of Object.values(db_handles)) {
+            deleteAsset(db, project, asset);
+        }
+        await fresh.internal_freshAsset(db_handles, project, asset, list_versions, find_latest, read_summary, read_metadata, tokenizable);
+
+    } else {
+        for (const db of Object.values(db_handles)) {
+            deleteVersion(db, project, asset, version);
+        }
+        const latest = find_latest(project, asset);
+        if (latest != null) { // short-circuit if latest = null, as this implies that there are no (non-probational) versions.
+            await fresh.internal_freshVersion(db_handles, project, asset, version, latest, read_summary, read_metadata, tokenizable);
+        }
+    }
+}

src/handlers/updateHandler.js

@@ -0,0 +1,92 @@
+import { addVersion } from "../sqlite/addVersion.js"; 
+import { deleteVersion } from "../sqlite/deleteVersion.js"; 
+import { deleteAsset } from "../sqlite/deleteAsset.js"; 
+import { deleteProject } from "../sqlite/deleteProject.js"; 
+import { setLatest } from "../sqlite/setLatest.js"; 
+import Database from "better-sqlite3"
+
+function safe_extract(x, p) {
+    if (p in x) {
+        return x[p];
+    } else {
+        throw new Error("expected a '" + p + "' property");
+    }
+}
+
+function is_latest(log) {
+    return "latest" in log && log.latest;
+}
+
+export async function updateHandler(db_paths, last_modified, read_logs, read_metadata, find_latest, tokenizable) {
+    const logs = await read_logs(last_modified);
+
+    // Need to make sure they're sorted so we execute the responses to the
+    // actions in the right order.
+    if (logs.length > 1) {
+        let sorted = true;
+        for (var i = 1; i < logs.length; ++i) {
+            if (logs[i].time < logs[i-1].time) {
+                sorted = false;
+            }
+        }
+        if (!sorted) {
+            logs.sort((a, b) => a.time - b.time);
+        }
+    }
+
+    const db_handles = {};
+    for (const [k, v] of Object.entries(db_paths)) {
+        db_handles[k] = Database(v);
+    }
+    const to_extract = Object.keys(db_handles);
+
+    for (const l of logs) {
+        const parameters = l.log;
+        const type = safe_extract(parameters, "type");
+
+        if (type == "add-version") {
+            const project = safe_extract(parameters, "project");
+            const asset = safe_extract(parameters, "asset");
+            const version = safe_extract(parameters, "version");
+            let output = await read_metadata(project, asset, version, to_extract);
+            for (const [e, db] of Object.entries(db_handles)) {
+                addVersion(db, project, asset, version, is_latest(parameters), output[e], tokenizable);
+            }
+
+        } else if (type == "delete-version") {
+            const project = safe_extract(parameters, "project");
+            const asset = safe_extract(parameters, "asset");
+            const version = safe_extract(parameters, "version");
+            for (const db of Object.values(db_handles)) {
+                deleteVersion(db, project, asset, version);
+            }
+
+            // If we just deleted the latest version, we need to reset the
+            // previous version with the latest information.
+            if (is_latest(parameters)) {
+                const latest = await find_latest(project, asset);
+                if (latest != null) {
+                    for (const db of Object.values(db_handles)) {
+                        setLatest(db, project, asset, latest);
+                    }
+                }
+            }
+
+        } else if (type == "delete-asset") {
+            const project = safe_extract(parameters, "project");
+            const asset = safe_extract(parameters, "asset");
+            for (const db of Object.values(db_handles)) {
+                deleteAsset(db, project, asset);
+            }
+
+        } else if (type == "delete-project") {
+            const project = safe_extract(parameters, "project");
+            for (const db of Object.values(db_handles)) {
+                deleteProject(db, project);
+            }
+
+        } else {
+            throw new Error("unknown update action type '" + type + "'");
+        }
+    }
+}

tests/handlers/freshHandler.test.js

@@ -0,0 +1,160 @@
+import * as path from "path";
+import * as utils from "../utils.js";
+import { freshHandler } from "../../src/handlers/freshHandler.js";
+import Database from "better-sqlite3";
+
+test("freshHandler works correctly without probation", async () => {
+    const testdir = utils.setupTestDirectory("freshHandler");
+    let all_paths = {};
+    for (const p of [ "_meta", "_alt" ]) {
+        all_paths[p] = path.join(testdir, "test" + p + ".sqlite3")
+    }
+
+    await freshHandler(
+        all_paths, 
+        () => [ "test", "retest" ],
+        project => {
+            if (project == "test") {
+                return ["foo"];
+            } else {
+                return ["whee", "stuff"];
+            }
+        },
+        (project, asset) => {
+            if (project == "test") {
+                return ["bar1", "bar2"];
+            } else {
+                return ["v1" ];
+            }
+        },
+        (project, asset) => {
+            if (project == "test") {
+                return "bar2";
+            } else {
+                return "v1";
+            }
+        },
+        (project, asset, version) => {
+            return {};
+        },
+        (project, asset, version, to_extract) => {
+            if (project == "test") {
+                return {
+                    "_meta": { "AAA.json": utils.mockMetadata["marcille"] },
+                    "_alt": { "BBB/CCC.txt": utils.mockMetadata["chicken"] }
+                }
+            } else {
+                return {
+                    "_meta": {
+                        "azur/CV.json": utils.mockMetadata["illustrious"],
+                        "thingy.csv": utils.mockMetadata["macrophage"],
+                    },
+                    "_alt": { "thingy.csv": utils.mockMetadata["macrophage"] }
+                }
+            }
+        },
+        new Set(["description", "motto"])
+    );
+
+    // Check that all versions are added, along with their metadata entries.
+    for (const [x, p] of Object.entries(all_paths)) {
+        const db = Database(p);
+
+        const vpayload = db.prepare("SELECT * FROM versions").all();
+        expect(vpayload.length).toBe(4);
+        expect(vpayload.map(x => x.project)).toEqual(["test", "test", "retest", "retest" ]);
+        expect(vpayload.map(x => x.asset)).toEqual(["foo", "foo", "whee", "stuff"]);
+        expect(vpayload.map(x => x.version)).toEqual(["bar1", "bar2", "v1", "v1"]);
+        expect(vpayload.map(x => x.latest)).toEqual([0, 1, 1, 1]);
+
+        let tpayload = db.prepare("SELECT * FROM tokens WHERE token = 'Donato'").all();
+        if (x == "_meta") {
+            expect(tpayload.length).toBeGreaterThan(0);
+        } else {
+            expect(tpayload.length).toEqual(0);
+        }
+
+        tpayload = db.prepare("SELECT * FROM tokens WHERE token = 'chicken'").all();
+        if (x == "_meta") {
+            expect(tpayload.length).toEqual(0);
+        } else {
+            expect(tpayload.length).toBeGreaterThan(0);
+        }
+
+        db.close();
+    }
+});
+
+test("freshHandler works correctly with probation", async () => {
+    const testdir = utils.setupTestDirectory("freshHandler");
+    let all_paths = {};
+    for (const p of [ "_meta", "_alt" ]) {
+        all_paths[p] = path.join(testdir, "test" + p + ".sqlite3")
+    }
+
+    await freshHandler(
+        all_paths, 
+        () => [ "test", "retest" ],
+        project => {
+            if (project == "test") {
+                return ["foo"];
+            } else {
+                return ["whee", "stuff"];
+            }
+        },
+        (project, asset) => {
+            if (project == "test") {
+                return ["bar1", "bar2"];
+            } else {
+                return ["v1" ];
+            }
+        },
+        (project, asset) => {
+            if (project == "test") {
+                return "bar1";
+            } else {
+                return null; // i.e., no non-probational versions.
+            }
+        },
+        (project, asset, version) => {
+            if (project == "test" && version == "bar2") {
+                return { on_probation: true }; // 'bar2' is not probational. 
+            } else {
+                return {};
+            }
+        },
+        (project, asset, version, to_extract) => {
+            return {
+                "_meta": { "AAA.json": utils.mockMetadata["marcille"] },
+                "_alt": { "BBB/CCC.txt": utils.mockMetadata["chicken"] }
+            }
+        },
+        new Set(["description"])
+    );
+
+    // Check that all versions are added, along with their metadata entries.
+    for (const [x, p] of Object.entries(all_paths)) {
+        const db = Database(p);
+
+        const vpayload = db.prepare("SELECT * FROM versions").all();
+        expect(vpayload.length).toBe(1);
+        expect(vpayload[0].project).toBe("test");
+        expect(vpayload[0].asset).toBe("foo");
+        expect(vpayload[0].version).toBe("bar1");
+        expect(vpayload[0].latest).toBe(1);
+
+        let tpayload = db.prepare("SELECT * FROM tokens WHERE token = 'Donato'").all();
+        if (x == "_meta") {
+            expect(tpayload.length).toBeGreaterThan(0);
+        } else {
+            expect(tpayload.length).toEqual(0);
+        }
+
+        tpayload = db.prepare("SELECT * FROM tokens WHERE token = 'chicken'").all();
+        if (x == "_meta") {
+            expect(tpayload.length).toEqual(0);
+        } else {
+            expect(tpayload.length).toBeGreaterThan(0);
+        }
+    }
+})