diff --git a/package.json b/package.json index 69b0562..f33743f 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "jest": "^29.3.1" }, "dependencies": { + "@aws-sdk/client-s3": "^3.470.0", "better-sqlite3": "^9.2.2" } } diff --git a/src/gypsum/fetchLatest.js b/src/gypsum/fetchLatest.js new file mode 100644 index 0000000..99817cc --- /dev/null +++ b/src/gypsum/fetchLatest.js @@ -0,0 +1,9 @@ +import { fetchJson } from "./utils.js"; + +export async function fetchLatest(url, project, asset) { + const found = await fetchJson(url, project + "/" + asset + "/..latest", { mustWork: false }); + if (found == null) { + return null; + } + return found.version; +} diff --git a/src/gypsum/index.js b/src/gypsum/index.js new file mode 100644 index 0000000..6eb07c9 --- /dev/null +++ b/src/gypsum/index.js @@ -0,0 +1,8 @@ +export * from "./fetchLatest.js"; +export * from "./listAssets.js"; +export * from "./listProjects.js"; +export * from "./listVersions.js"; +export * from "./listLogs.js"; +export * from "./readLog.js"; +export * from "./readMetadata.js"; +export * from "./readSummary.js"; diff --git a/src/gypsum/listAssets.js b/src/gypsum/listAssets.js new file mode 100644 index 0000000..847ea37 --- /dev/null +++ b/src/gypsum/listAssets.js @@ -0,0 +1,28 @@ +import { quickList } from "./utils.js"; + +export async function listAssets(url, project) { + let prefix = project + "/"; + let options = { Prefix: prefix, Delimiter: "/" }; + + let accumulated = []; + await quickList(url, options, resp => { + if ("CommonPrefixes" in resp) { + for (const x of resp.CommonPrefixes) { + let y = x.Prefix; + let i = y.lastIndexOf("/"); + if (i >= 0) { + y = y.slice(prefix.length, i); + } else { + y = y.slice(prefix.length); + } + + if (y.startsWith("..")) { + continue + } + accumulated.push(y); + } + } + }); + + return accumulated; +} diff --git a/src/gypsum/listLogs.js b/src/gypsum/listLogs.js new file mode 100644 index 0000000..30e6f7c --- /dev/null +++ b/src/gypsum/listLogs.js @@ -0,0 +1,20 @@ +import { quickList } from "./utils.js"; + +export async function listLogs(url, threshold) { + const prefix = "..logs/"; + + let accumulated = []; + await quickList( + url, + { Prefix: prefix, StartAfter: prefix + threshold }, + resp => { + if ("Contents" in resp) { + for (const x of resp.Contents) { + accumulated.push(x.Key.slice(prefix.length)); + } + } + } + ); + + return accumulated; +} diff --git a/src/gypsum/listProjects.js b/src/gypsum/listProjects.js new file mode 100644 index 0000000..d1c6f44 --- /dev/null +++ b/src/gypsum/listProjects.js @@ -0,0 +1,24 @@ +import { quickList } from "./utils.js"; + +export async function listProjects(url) { + let options = { Delimiter: "/" }; + + let accumulated = []; + await quickList(url, options, resp => { + if ("CommonPrefixes" in resp) { + for (const x of resp.CommonPrefixes) { + let y = x.Prefix; + if (y.startsWith("..")) { + continue + } + let i = y.indexOf("/"); + if (i >= 0) { + y = y.slice(0, i); + } + accumulated.push(y); + } + } + }); + + return accumulated; +} diff --git a/src/gypsum/listVersions.js b/src/gypsum/listVersions.js new file mode 100644 index 0000000..e1b6d4d --- /dev/null +++ b/src/gypsum/listVersions.js @@ -0,0 +1,28 @@ +import { quickList } from "./utils.js"; + +export async function listVersions(url, project, asset) { + let prefix = project + "/" + asset + "/"; + let options = { Prefix: prefix, Delimiter: "/" }; + + let accumulated = []; + await quickList(url, options, resp => { + if ("CommonPrefixes" in resp) { + for (const x of resp.CommonPrefixes) { + let y = x.Prefix; + let i = y.lastIndexOf("/"); + if (i >= 0) { + y = y.slice(prefix.length, i); + } else { + y = y.slice(prefix.length); + } + + if (y.startsWith("..")) { + continue + } + accumulated.push(y); + } + } + }); + + return accumulated; +} diff --git a/src/gypsum/readLog.js b/src/gypsum/readLog.js new file mode 100644 index 0000000..ac53a8f --- /dev/null +++ b/src/gypsum/readLog.js @@ -0,0 +1,5 @@ +import { fetchJson } from "./utils.js"; + +export function readLog(url, name) { + return fetchJson(url, "..logs/" + name); +} diff --git a/src/gypsum/readMetadata.js b/src/gypsum/readMetadata.js new file mode 100644 index 0000000..14009f3 --- /dev/null +++ b/src/gypsum/readMetadata.js @@ -0,0 +1,34 @@ +import { fetchFile, fetchJson } from "./utils.js"; + +export async function readMetadata(url, project, asset, version, to_extract, { parse = true } = {}) { + const output = {}; + for (const s of to_extract) { + output[s] = {}; + } + + const fun = (parse ? fetchJson : (url, key) => fetchFile(url, key).then(x => x.Body.transformToString("utf-8"))); + + let manifest = await fetchJson(url, project + "/" + asset + "/" + version + "/..manifest"); + for (const [k, v] of Object.entries(manifest)) { + let i = k.lastIndexOf("/"); + let base = (i < 0 ? k : k.slice(i + 1)); + if (base in output) { + let dir = (i < 0 ? "." : k.slice(0, i)); + + let key; + if ("link" in v) { + let target = v.link; + if ("ancestor" in target) { + target = target.ancestor; + } + key = target.project + "/" + target.asset + "/" + target.version + "/" + target.path; + } else { + key = project + "/" + asset + "/" + version + "/" + k; + } + + output[base][dir] = await fun(url, key); + } + } + + return output; +} diff --git a/src/gypsum/readSummary.js b/src/gypsum/readSummary.js new file mode 100644 index 0000000..f793a87 --- /dev/null +++ b/src/gypsum/readSummary.js @@ -0,0 +1,5 @@ +import { fetchJson } from "./utils.js"; + +export function readSummary(url, project, asset, version) { + return fetchJson(url, project + "/" + asset + "/" + version + "/..summary"); +} diff --git a/src/gypsum/utils.js b/src/gypsum/utils.js new file mode 100644 index 0000000..4e0af50 --- /dev/null +++ b/src/gypsum/utils.js @@ -0,0 +1,64 @@ +import { ListObjectsV2Command, GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; + +const cached = {}; + +export async function setupS3(url) { + if (!(url in cached)) { + let res = await fetch(url + "/credentials/s3-api"); + if (!res.ok) { + throw new Error("failed to retrieve S3 credentials for bucket access") + } + + let config = await res.json(); + + const client = new S3Client({ + region: "auto", + endpoint: config.endpoint, + credentials: { + accessKeyId: config.key, + secretAccessKey: config.secret + } + }); + + cached[url] = { bucket: config.bucket, client }; + } + + return cached[url]; +} + +export async function quickList(url, params, fun) { + const { bucket, client } = await setupS3(url); + let options = { Bucket: bucket, ...params }; + while (true) { + let out = await client.send(new ListObjectsV2Command(options)); + fun(out); + if (!out.IsTruncated) { + break; + } + options.ContinuationToken = out.NextContinuationToken; + } +} + +export async function fetchFile(url, key, { mustWork = true } = {}) { + const { bucket, client } = await setupS3(url); + + try { + // Need the await here for the try/catch to work properly. + return await client.send(new GetObjectCommand({Bucket: bucket, Key: key })); + } catch (e) { + if (mustWork) { + throw new Error("failed to fetch '" + key + "'", { cause: e }); + } else { + return null; + } + } +} + +export async function fetchJson(url, key, { mustWork = true } = {}) { + let fres = await fetchFile(url, key, { mustWork }); + if (fres === null) { + return null; + } + let stringified = await fres.Body.transformToString("utf-8"); + return JSON.parse(stringified); +} diff --git a/tests/gypsum/fetchLatest.test.js b/tests/gypsum/fetchLatest.test.js new file mode 100644 index 0000000..d392224 --- /dev/null +++ b/tests/gypsum/fetchLatest.test.js @@ -0,0 +1,11 @@ +import * as utils from "./utils.js"; +import { fetchLatest } from "../../src/gypsum/fetchLatest.js"; + +test("fetchLatest works as expected", async () => { + let v = await fetchLatest(utils.testUrl, "test-R", "basic"); + expect(v).not.toBeNull(); + + // Just returns null if the ..latest file doesn't exist. + let v0 = await fetchLatest(utils.testUrl, "test-R", "asset_does_not_exist"); + expect(v0).toBeNull(); +}); diff --git a/tests/gypsum/listAssets.test.js b/tests/gypsum/listAssets.test.js new file mode 100644 index 0000000..e80261b --- /dev/null +++ b/tests/gypsum/listAssets.test.js @@ -0,0 +1,8 @@ +import * as utils from "./utils.js"; +import { listAssets } from "../../src/gypsum/listAssets.js"; + +test("listAssets works as expected", async () => { + let contents = await listAssets(utils.testUrl, "test-R"); + expect(contents.length).toBeGreaterThan(0); + expect(contents.indexOf("basic")).toBeGreaterThanOrEqual(0); +}); diff --git a/tests/gypsum/listLogs.test.js b/tests/gypsum/listLogs.test.js new file mode 100644 index 0000000..6dc3a3e --- /dev/null +++ b/tests/gypsum/listLogs.test.js @@ -0,0 +1,12 @@ +import * as utils from "./utils.js"; +import { listLogs } from "../../src/gypsum/listLogs.js"; + +test("listLogs works as expected", async () => { + const threshold = (new Date(Date.now() - 100 * 24 * 60 * 60 * 1000)).toISOString(); + + // Can't really do a lot of tests here, as we don't know what the current logs are. + let all_logs = await listLogs(utils.testUrl, threshold); + for (const x of all_logs) { + expect(x >= threshold).toBe(true); + } +}); diff --git a/tests/gypsum/listProjects.test.js b/tests/gypsum/listProjects.test.js new file mode 100644 index 0000000..3a26272 --- /dev/null +++ b/tests/gypsum/listProjects.test.js @@ -0,0 +1,8 @@ +import * as utils from "./utils.js"; +import { listProjects } from "../../src/gypsum/listProjects.js"; + +test("listProjects works as expected", async () => { + let contents = await listProjects(utils.testUrl); + expect(contents.length).toBeGreaterThan(0); + expect(contents.indexOf("test-R")).toBeGreaterThanOrEqual(0); +}) diff --git a/tests/gypsum/listVersions.test.js b/tests/gypsum/listVersions.test.js new file mode 100644 index 0000000..aa6eb36 --- /dev/null +++ b/tests/gypsum/listVersions.test.js @@ -0,0 +1,8 @@ +import * as utils from "./utils.js"; +import { listVersions } from "../../src/gypsum/listVersions.js"; + +test("listVersions works as expected", async () => { + let contents = await listVersions(utils.testUrl, "test-R", "basic"); + expect(contents.length).toBeGreaterThan(0); + expect(contents.indexOf("v1")).toBeGreaterThanOrEqual(0); +}); diff --git a/tests/gypsum/readLog.test.js b/tests/gypsum/readLog.test.js new file mode 100644 index 0000000..0389da1 --- /dev/null +++ b/tests/gypsum/readLog.test.js @@ -0,0 +1,12 @@ +import * as utils from "./utils.js"; +import { listLogs } from "../../src/gypsum/listLogs.js"; +import { readLog } from "../../src/gypsum/readLog.js"; + +test("readLog works as expected", async () => { + // Hard to say if a lot is actually present, but if it is, we'll try to load it in. + let all_logs = await listLogs(utils.testUrl, "zzzz"); + if (all_logs.length) { + const out = readLog(utils.testUrl, all_logs[0]) + expect(typeof out.type).toBe("string"); + } +}); diff --git a/tests/gypsum/readMetadata.test.js b/tests/gypsum/readMetadata.test.js new file mode 100644 index 0000000..0029795 --- /dev/null +++ b/tests/gypsum/readMetadata.test.js @@ -0,0 +1,39 @@ +import { readMetadata } from "../../src/gypsum/readMetadata.js"; +import * as utils from "./utils.js"; + +test("readMetadata works in the simple case", async () => { + let contents = await readMetadata(utils.testUrl, "test-R", "basic", "v1", [ "blah.txt", "bar.txt" ], { parse: false }); + let sub = contents["blah.txt"]; + expect("." in sub).toBe(true); + expect(typeof sub["."]).toBe("string"); + + sub = contents["bar.txt"]; + expect("foo" in sub).toBe(true); + expect(typeof sub["foo"]).toBe("string"); +}) + +test("readMetadata works with links", async () => { + // v2 is linked to v1. + { + let contents = await readMetadata(utils.testUrl, "test-R", "basic", "v2", [ "blah.txt", "bar.txt" ], { parse: false }); + let sub = contents["blah.txt"]; + expect("." in sub).toBe(true); + expect(typeof sub["."]).toBe("string"); + + sub = contents["bar.txt"]; + expect("foo" in sub).toBe(true); + expect(typeof sub["foo"]).toBe("string"); + } + + // v3 is linked to v1 via v2. + { + let contents = await readMetadata(utils.testUrl, "test-R", "basic", "v3", [ "blah.txt", "bar.txt" ], { parse: false }); + let sub = contents["blah.txt"]; + expect("." in sub).toBe(true); + expect(typeof sub["."]).toBe("string"); + + sub = contents["bar.txt"]; + expect("foo" in sub).toBe(true); + expect(typeof sub["foo"]).toBe("string"); + } +}); diff --git a/tests/gypsum/readSummary.test.js b/tests/gypsum/readSummary.test.js new file mode 100644 index 0000000..9173644 --- /dev/null +++ b/tests/gypsum/readSummary.test.js @@ -0,0 +1,7 @@ +import * as utils from "./utils.js"; +import { readSummary } from "../../src/gypsum/readSummary.js"; + +test("fetchLatest works as expected", async () => { + let v = await readSummary(utils.testUrl, "test-R", "basic", "v1"); + expect(typeof v.upload_user_id).toBe("string"); +}); diff --git a/tests/gypsum/utils.js b/tests/gypsum/utils.js new file mode 100644 index 0000000..95aee5f --- /dev/null +++ b/tests/gypsum/utils.js @@ -0,0 +1 @@ +export const testUrl = "https://gypsum.artifactdb.com";