diff --git a/index.test.ts b/index.test.ts index b398ad0..44ecef7 100644 --- a/index.test.ts +++ b/index.test.ts @@ -5,7 +5,7 @@ import { Env } from "."; import * as fetchAuth from "./index"; import { RegistryTokens } from "./src/token"; import { RegistryAuthProtocolTokenPayload } from "./src/auth"; -import { registries } from "./src/registry/registry"; +import { ListRepositoriesResponse, registries } from "./src/registry/registry"; import { RegistryHTTPClient } from "./src/registry/http"; function createRequest(method: string, path: string, body: ReadableStream | null, headers = {}) { @@ -112,6 +112,7 @@ describe("v2 manifests", () => { "content-type": "application/gzip", "docker-content-digest": sha256, }); + await bindings.REGISTRY.delete(`${name}/manifests/${reference}`); }); test("PUT /v2/:name/manifests/:reference works", () => createManifest("hello-world-main", "{}", "hello")); @@ -371,3 +372,50 @@ describe("http client", () => { expect("exists" in res && res.exists).toBe(false); }); }); + +describe("push and catalog", () => { + test("push and then use the catalog", async () => { + await createManifest("hello-world-main", "{}", "hello"); + await createManifest("hello-world-main", "{}", "latest"); + await createManifest("hello-world-main", "{}", "hello-2"); + await createManifest("hello", "{}", "hello"); + await createManifest("hello/hello", "{}", "hello"); + + const response = await fetchUnauth(createRequest("GET", "/v2/_catalog", null)); + expect(response.ok).toBeTruthy(); + const body = (await response.json()) as { repositories: string[] }; + expect(body).toEqual({ + repositories: ["hello-world-main", "hello/hello", "hello"], + }); + const expectedRepositories = body.repositories; + const tagsRes = await fetchUnauth(createRequest("GET", `/v2/hello-world-main/tags/list?n=1000`, null)); + const tags = (await tagsRes.json()) as TagsList; + expect(tags.name).toEqual("hello-world-main"); + expect(tags.tags).toEqual([ + "hello", + "hello-2", + "latest", + "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + ]); + + const repositoryBuildUp: string[] = []; + let currentPath = "/v2/_catalog?n=1"; + for (let i = 0; i < 3; i++) { + const response = await fetchUnauth(createRequest("GET", currentPath, null)); + expect(response.ok).toBeTruthy(); + const body = (await response.json()) as { repositories: string[] }; + if (body.repositories.length === 0) { + break; + } + expect(body.repositories).toHaveLength(1); + + repositoryBuildUp.push(...body.repositories); + const url = new URL(response.headers.get("Link")!.split(";")[0].trim()); + console.log(url.pathname, url.search, response.headers.get("Link")); + currentPath = url.pathname + url.search; + console.log(currentPath); + } + + expect(repositoryBuildUp).toEqual(expectedRepositories); + }); +}); diff --git a/src/errors.ts b/src/errors.ts index 473dd44..9dadb5d 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -24,14 +24,28 @@ export class AuthErrorResponse extends Response { export class RangeError extends Response { constructor(stateStr: string, state: State) { - super("{}", { - status: 416, - headers: { - "Location": `/v2/${state.name}/blobs/uploads/${state.registryUploadId}?_state=${stateStr}`, - "Range": `0-${state.byteRange - 1}`, - "Docker-Upload-UUID": state.registryUploadId, + super( + JSON.stringify({ + errors: [ + { + code: "RANGE_ERROR", + message: `state ${stateStr} is not satisfiable (upload id: ${state.registryUploadId})`, + detail: { + ...state, + string: stateStr, + }, + }, + ], + }), + { + status: 416, + headers: { + "Location": `/v2/${state.name}/blobs/uploads/${state.registryUploadId}?_state=${stateStr}`, + "Range": `0-${state.byteRange - 1}`, + "Docker-Upload-UUID": state.registryUploadId, + }, }, - }); + ); } } diff --git a/src/registry/http.ts b/src/registry/http.ts index d602adf..1bb7c3a 100644 --- a/src/registry/http.ts +++ b/src/registry/http.ts @@ -7,6 +7,7 @@ import { FinishedUploadObject, GetLayerResponse, GetManifestResponse, + ListRepositoriesResponse, PutManifestResponse, Registry, RegistryConfiguration, @@ -64,7 +65,6 @@ function ctxIntoRequest(ctx: HTTPContext, url: URL, method: string, path: string const urlReq = `${url.protocol}//${ctx.authContext.service}/v2${ ctx.repository === "" ? "/" : ctx.repository + "/" }${path}`; - console.log("Doing request:", urlReq); return new Request(urlReq, { method, body, @@ -467,6 +467,10 @@ export class RegistryHTTPClient implements Registry { ): Promise { throw new Error("unimplemented"); } + + async listRepositories(_limit?: number, _last?: string): Promise { + throw new Error("unimplemented"); + } } // AuthType defined the supported auth types diff --git a/src/registry/r2.ts b/src/registry/r2.ts index d2acc43..af8bdd9 100644 --- a/src/registry/r2.ts +++ b/src/registry/r2.ts @@ -132,30 +132,97 @@ export class R2Registry implements Registry { } async listRepositories(limit?: number, last?: string): Promise { - const env = this.env; + // The idea in listRepositories is list all entries in the R2 bucket and map them to repositories. + // We do this by taking advantage of the name format in the R2 bucket: + // name format is: + // /<'blobs' | 'manifests'>/ + // This means we slice the last two items in the key and add them to our hash map. + // At the end, we start skipping entries until we find another unique key, then we return that entry as startAfter. + const options = { - limit: limit ? limit : 1000, - delimiter: "/", - startAfter: last, - } - const r2Objects = (await env.REGISTRY.list(options)); + limit: limit ?? 1000, + startAfter: last ?? undefined, + }; + const repositories: Record = {}; + let totalRecords = 0; + let lastSeen: string | undefined; + const objectExistsInPath = (entry: string) => { + const parts = entry.split("/"); + const repository = parts.slice(0, parts.length - 2).join("/"); + return repository in repositories; + }; - let truncated = r2Objects.truncated; - let cursor = truncated ? r2Objects.cursor : undefined; + const addObjectPath = (object: R2Object) => { + // update lastSeen for cursoring purposes + lastSeen = object.key; + // don't add if seen before + if (totalRecords >= options.limit) return; + // skip either 'manifests' or 'blobs' + // name format is: + // /<'blobs' | 'manifests'>/ + const parts = object.key.split("/"); + const repository = parts.slice(0, parts.length - 2).join("/"); + if (!(repository in repositories)) { + totalRecords++; + } - while (truncated) { - const next = await env.REGISTRY.list({ - ...options, - cursor: cursor, + repositories[repository] = {}; + }; + + const r2Objects = await this.env.REGISTRY.list({ + limit: options.limit, + startAfter: options.startAfter, + }); + r2Objects.objects.forEach((path) => addObjectPath(path)); + let cursor = r2Objects.truncated ? r2Objects.cursor : undefined; + while (cursor !== undefined && totalRecords < options.limit) { + const next = await this.env.REGISTRY.list({ + limit: options.limit, + cursor, }); - r2Objects.objects.push(...next.objects); - - truncated = next.truncated; - cursor = next.cursor + next.objects.forEach((path) => addObjectPath(path)); + if (next.truncated) { + cursor = next.cursor; + } else { + cursor = undefined; + } + } + + while (cursor !== undefined && typeof lastSeen === "string" && objectExistsInPath(lastSeen)) { + const nextList: R2Objects = await this.env.REGISTRY.list({ + limit: 1000, + cursor, + }); + + let found = false; + // Search for the next object in the list + for (const object of nextList.objects) { + lastSeen = object.key; + if (!objectExistsInPath(lastSeen)) { + found = true; + break; + } + } + + if (found) break; + + if (nextList.truncated) { + // jump to the next list and try to find a + // repository that hasn't been returned in this response + cursor = nextList.cursor; + } else { + // we arrived to the end of the list, no more cursor + cursor = undefined; + } + } + + if (cursor === undefined) { + lastSeen = undefined; } return { - repositories: r2Objects.delimitedPrefixes.map((name)=> name.endsWith('/') ? name.slice(0, -1) : name) + repositories: Object.keys(repositories), + cursor: lastSeen, }; } diff --git a/src/registry/registry.ts b/src/registry/registry.ts index e115bd6..49643a6 100644 --- a/src/registry/registry.ts +++ b/src/registry/registry.ts @@ -43,11 +43,10 @@ export type CheckManifestResponse = exists: false; }; -export type ListRepositoriesResponse = - { - repositories: string[]; - } - +export type ListRepositoriesResponse = { + repositories: string[]; + cursor?: string; +}; // Response layerExists call export type CheckLayerResponse = diff --git a/src/router.ts b/src/router.ts index 2eabac6..e06daf6 100644 --- a/src/router.ts +++ b/src/router.ts @@ -27,15 +27,27 @@ v2Router.get("/", async (_req, _env: Env) => { v2Router.get("/_catalog", async (req, env: Env) => { const { n, last } = req.query; - const res = await env.REGISTRY_CLIENT.listRepositories( + const response = await env.REGISTRY_CLIENT.listRepositories( n ? parseInt(n?.toString()) : undefined, - last?.toString() + last?.toString(), ); + if ("response" in response) { + return response.response; + } - return new Response(JSON.stringify(res)); + const url = new URL(req.url); + return new Response( + JSON.stringify({ + repositories: response.repositories, + }), + { + headers: { + Link: `${url.protocol}//${url.hostname}${url.pathname}?n=${n ?? 1000}&last=${response.cursor ?? ""}; rel=next`, + }, + }, + ); }); - v2Router.delete("/:name+/manifests/:reference", async (req, env: Env) => { // deleting a manifest works by retrieving the """main""" manifest that its key is a sha, // and then going through every tag and removing it @@ -49,17 +61,17 @@ v2Router.delete("/:name+/manifests/:reference", async (req, env: Env) => { // // If somehow we need to remove by paginating, we accept a last query param - const { last } = req.query; + const { last, limit } = req.query; const { name, reference } = req.params; // Reference is ALWAYS a sha256 const manifest = await env.REGISTRY.head(`${name}/manifests/${reference}`); if (manifest === null) { return new Response(JSON.stringify(ManifestUnknownError), { status: 404 }); } - + const limitInt = parseInt(limit?.toString() ?? "1000", 10); const tags = await env.REGISTRY.list({ prefix: `${name}/manifests`, - limit: 1000, + limit: isNaN(limitInt) ? 1000 : limitInt, startAfter: last?.toString(), }); for (const tag of tags.objects) { @@ -76,7 +88,7 @@ v2Router.delete("/:name+/manifests/:reference", async (req, env: Env) => { return new Response(JSON.stringify(ManifestTagsListTooBigError), { status: 400, headers: { - "Link": `${req.url}/last=${tags.objects[tags.objects.length - 1]}; rel=next`, + "Link": `${req.url}/?last=${tags.truncated ? tags.cursor : ""}; rel=next`, "Content-Type": "application/json", }, });