Migrated functions for querying the gypsum bucket with S3 credentials. (

#3)

This provides methods for the abstract listing/reading generics, pulling from the
gypsum R2 bucket instead of inspecting the registry on the local filesystem.

package.json

@@ -14,6 +14,7 @@
     "jest": "^29.3.1"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.470.0",
     "better-sqlite3": "^9.2.2"
   }
 }

src/gypsum/fetchLatest.js

@@ -0,0 +1,9 @@
+import { fetchJson } from "./utils.js";
+
+export async function fetchLatest(url, project, asset) {
+    const found = await fetchJson(url, project + "/" + asset + "/..latest", { mustWork: false });
+    if (found == null) {
+        return null;
+    }
+    return found.version;
+}

src/gypsum/index.js

@@ -0,0 +1,8 @@
+export * from "./fetchLatest.js";
+export * from "./listAssets.js";
+export * from "./listProjects.js";
+export * from "./listVersions.js";
+export * from "./listLogs.js";
+export * from "./readLog.js";
+export * from "./readMetadata.js";
+export * from "./readSummary.js";

src/gypsum/listAssets.js

@@ -0,0 +1,28 @@
+import { quickList } from "./utils.js";
+
+export async function listAssets(url, project) {
+    let prefix = project + "/";
+    let options = { Prefix: prefix, Delimiter: "/" };
+
+    let accumulated = [];
+    await quickList(url, options, resp => {
+        if ("CommonPrefixes" in resp) {
+            for (const x of resp.CommonPrefixes) {
+                let y = x.Prefix;
+                let i = y.lastIndexOf("/");
+                if (i >= 0) {
+                    y = y.slice(prefix.length, i);
+                } else {
+                    y = y.slice(prefix.length);
+                }
+
+                if (y.startsWith("..")) {
+                    continue
+                }
+                accumulated.push(y);
+            }
+        }
+    });
+
+    return accumulated;
+}

src/gypsum/listLogs.js

@@ -0,0 +1,20 @@
+import { quickList } from "./utils.js";
+
+export async function listLogs(url, threshold) {
+    const prefix = "..logs/";
+
+    let accumulated = [];
+    await quickList(
+        url,
+        { Prefix: prefix, StartAfter: prefix + threshold }, 
+        resp => {
+            if ("Contents" in resp) {
+                for (const x of resp.Contents) {
+                    accumulated.push(x.Key.slice(prefix.length));
+                }
+            }
+        }
+    );
+
+    return accumulated;
+}

src/gypsum/listProjects.js

@@ -0,0 +1,24 @@
+import { quickList } from "./utils.js";
+
+export async function listProjects(url) {
+    let options = { Delimiter: "/" };
+
+    let accumulated = [];
+    await quickList(url, options, resp => {
+        if ("CommonPrefixes" in resp) {
+            for (const x of resp.CommonPrefixes) {
+                let y = x.Prefix;
+                if (y.startsWith("..")) {
+                    continue
+                }
+                let i = y.indexOf("/");
+                if (i >= 0) {
+                    y = y.slice(0, i);
+                }
+                accumulated.push(y);
+            }
+        }
+    });
+
+    return accumulated;
+}

src/gypsum/listVersions.js

@@ -0,0 +1,28 @@
+import { quickList } from "./utils.js";
+
+export async function listVersions(url, project, asset) {
+    let prefix = project + "/" + asset + "/";
+    let options = { Prefix: prefix, Delimiter: "/" };
+
+    let accumulated = [];
+    await quickList(url, options, resp => {
+        if ("CommonPrefixes" in resp) {
+            for (const x of resp.CommonPrefixes) {
+                let y = x.Prefix;
+                let i = y.lastIndexOf("/");
+                if (i >= 0) {
+                    y = y.slice(prefix.length, i);
+                } else {
+                    y = y.slice(prefix.length);
+                }
+
+                if (y.startsWith("..")) {
+                    continue
+                }
+                accumulated.push(y);
+            }
+        }
+    });
+
+    return accumulated;
+}

src/gypsum/readLog.js

@@ -0,0 +1,5 @@
+import { fetchJson } from "./utils.js";
+
+export function readLog(url, name) {
+    return fetchJson(url, "..logs/" + name);
+}

src/gypsum/readMetadata.js

@@ -0,0 +1,34 @@
+import { fetchFile, fetchJson } from "./utils.js";
+
+export async function readMetadata(url, project, asset, version, to_extract, { parse = true } = {}) {
+    const output = {};
+    for (const s of to_extract) {
+        output[s] = {};
+    }
+
+    const fun = (parse ? fetchJson : (url, key) => fetchFile(url, key).then(x => x.Body.transformToString("utf-8")));
+
+    let manifest = await fetchJson(url, project + "/" + asset + "/" + version + "/..manifest");
+    for (const [k, v] of Object.entries(manifest)) {
+        let i = k.lastIndexOf("/");
+        let base = (i < 0 ? k : k.slice(i + 1));
+        if (base in output) {
+            let dir = (i < 0 ? "." : k.slice(0, i));
+
+            let key;
+            if ("link" in v) {
+                let target = v.link;
+                if ("ancestor" in target) {
+                    target = target.ancestor;
+                }
+                key = target.project + "/" + target.asset + "/" + target.version + "/" + target.path;
+            } else {
+                key = project + "/" + asset + "/" + version + "/" + k;
+            }
+
+            output[base][dir] = await fun(url, key);
+        }
+    }
+
+    return output;
+}

src/gypsum/readSummary.js

@@ -0,0 +1,5 @@
+import { fetchJson } from "./utils.js";
+
+export function readSummary(url, project, asset, version) {
+    return fetchJson(url, project + "/" + asset + "/" + version + "/..summary");
+}

src/gypsum/utils.js

@@ -0,0 +1,64 @@
+import { ListObjectsV2Command, GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
+
+const cached = {};
+
+export async function setupS3(url) {
+    if (!(url in cached)) {
+        let res = await fetch(url + "/credentials/s3-api");
+        if (!res.ok) {
+            throw new Error("failed to retrieve S3 credentials for bucket access")
+        }
+
+        let config = await res.json();
+
+        const client = new S3Client({
+            region: "auto",
+            endpoint: config.endpoint,
+            credentials: {
+                accessKeyId: config.key,
+                secretAccessKey: config.secret
+            }
+        });
+
+        cached[url] = { bucket: config.bucket, client };
+    }
+
+    return cached[url];
+}
+
+export async function quickList(url, params, fun) {
+    const { bucket, client } = await setupS3(url);
+    let options = { Bucket: bucket, ...params };
+    while (true) {
+        let out = await client.send(new ListObjectsV2Command(options));
+        fun(out);
+        if (!out.IsTruncated) {
+            break;
+        }
+        options.ContinuationToken = out.NextContinuationToken;
+    }
+}
+
+export async function fetchFile(url, key, { mustWork = true } = {}) {
+    const { bucket, client } = await setupS3(url);
+
+    try {
+        // Need the await here for the try/catch to work properly.
+        return await client.send(new GetObjectCommand({Bucket: bucket, Key: key }));
+    } catch (e) {
+        if (mustWork) {
+            throw new Error("failed to fetch '" + key + "'", { cause: e });
+        } else {
+            return null;
+        }
+    }
+}
+
+export async function fetchJson(url, key, { mustWork = true } = {}) {
+    let fres = await fetchFile(url, key, { mustWork });
+    if (fres === null) {
+        return null;
+    }
+    let stringified = await fres.Body.transformToString("utf-8");
+    return JSON.parse(stringified);
+}

tests/gypsum/fetchLatest.test.js

@@ -0,0 +1,11 @@
+import * as utils from "./utils.js";
+import { fetchLatest } from "../../src/gypsum/fetchLatest.js";
+
+test("fetchLatest works as expected", async () => {
+    let v = await fetchLatest(utils.testUrl, "test-R", "basic");
+    expect(v).not.toBeNull();
+
+    // Just returns null if the ..latest file doesn't exist.
+    let v0 = await fetchLatest(utils.testUrl, "test-R", "asset_does_not_exist");
+    expect(v0).toBeNull();
+});

tests/gypsum/listAssets.test.js

@@ -0,0 +1,8 @@
+import * as utils from "./utils.js";
+import { listAssets } from "../../src/gypsum/listAssets.js";
+
+test("listAssets works as expected", async () => {
+    let contents = await listAssets(utils.testUrl, "test-R");
+    expect(contents.length).toBeGreaterThan(0);
+    expect(contents.indexOf("basic")).toBeGreaterThanOrEqual(0);
+});

tests/gypsum/listLogs.test.js

@@ -0,0 +1,12 @@
+import * as utils from "./utils.js";
+import { listLogs } from "../../src/gypsum/listLogs.js";
+
+test("listLogs works as expected", async () => {
+    const threshold = (new Date(Date.now() - 100 * 24 * 60 * 60 * 1000)).toISOString();
+
+    // Can't really do a lot of tests here, as we don't know what the current logs are.
+    let all_logs = await listLogs(utils.testUrl, threshold);
+    for (const x of all_logs) {
+        expect(x >= threshold).toBe(true);
+    }
+});

tests/gypsum/listProjects.test.js

@@ -0,0 +1,8 @@
+import * as utils from "./utils.js";
+import { listProjects } from "../../src/gypsum/listProjects.js";
+
+test("listProjects works as expected", async () => {
+    let contents = await listProjects(utils.testUrl);
+    expect(contents.length).toBeGreaterThan(0);
+    expect(contents.indexOf("test-R")).toBeGreaterThanOrEqual(0);
+})

tests/gypsum/listVersions.test.js

@@ -0,0 +1,8 @@
+import * as utils from "./utils.js";
+import { listVersions } from "../../src/gypsum/listVersions.js";
+
+test("listVersions works as expected", async () => {
+    let contents = await listVersions(utils.testUrl, "test-R", "basic");
+    expect(contents.length).toBeGreaterThan(0);
+    expect(contents.indexOf("v1")).toBeGreaterThanOrEqual(0);
+});

tests/gypsum/readLog.test.js

@@ -0,0 +1,12 @@
+import * as utils from "./utils.js";
+import { listLogs } from "../../src/gypsum/listLogs.js";
+import { readLog } from "../../src/gypsum/readLog.js";
+
+test("readLog works as expected", async () => {
+    // Hard to say if a lot is actually present, but if it is, we'll try to load it in.
+    let all_logs = await listLogs(utils.testUrl, "zzzz");
+    if (all_logs.length) {
+        const out = readLog(utils.testUrl, all_logs[0])
+        expect(typeof out.type).toBe("string");
+    }
+});

tests/gypsum/readMetadata.test.js

@@ -0,0 +1,39 @@
+import { readMetadata } from "../../src/gypsum/readMetadata.js";
+import * as utils from "./utils.js";
+
+test("readMetadata works in the simple case", async () => {
+    let contents = await readMetadata(utils.testUrl, "test-R", "basic", "v1", [ "blah.txt", "bar.txt" ], { parse: false });
+    let sub = contents["blah.txt"];
+    expect("." in sub).toBe(true);
+    expect(typeof sub["."]).toBe("string");
+
+    sub = contents["bar.txt"];
+    expect("foo" in sub).toBe(true);
+    expect(typeof sub["foo"]).toBe("string");
+})
+
+test("readMetadata works with links", async () => {
+    // v2 is linked to v1.
+    {
+        let contents = await readMetadata(utils.testUrl, "test-R", "basic", "v2", [ "blah.txt", "bar.txt" ], { parse: false });
+        let sub = contents["blah.txt"];
+        expect("." in sub).toBe(true);
+        expect(typeof sub["."]).toBe("string");
+
+        sub = contents["bar.txt"];
+        expect("foo" in sub).toBe(true);
+        expect(typeof sub["foo"]).toBe("string");
+    }
+
+    // v3 is linked to v1 via v2.
+    {
+        let contents = await readMetadata(utils.testUrl, "test-R", "basic", "v3", [ "blah.txt", "bar.txt" ], { parse: false });
+        let sub = contents["blah.txt"];
+        expect("." in sub).toBe(true);
+        expect(typeof sub["."]).toBe("string");
+
+        sub = contents["bar.txt"];
+        expect("foo" in sub).toBe(true);
+        expect(typeof sub["foo"]).toBe("string");
+    }
+});

tests/gypsum/readSummary.test.js

@@ -0,0 +1,7 @@
+import * as utils from "./utils.js";
+import { readSummary } from "../../src/gypsum/readSummary.js";
+
+test("fetchLatest works as expected", async () => {
+    let v = await readSummary(utils.testUrl, "test-R", "basic", "v1");
+    expect(typeof v.upload_user_id).toBe("string");
+});

tests/gypsum/utils.js

@@ -0,0 +1 @@
+export const testUrl = "https://gypsum.artifactdb.com";