fix: type errors and lints in listManifests

We also need to do a tweak in listManifests where we are able to return
a repository with multiple components in the path.

Added a unit test to showcase it works.

Also adding more context to RangeError.

index.test.ts

@@ -5,7 +5,7 @@ import { Env } from ".";
 import * as fetchAuth from "./index";
 import { RegistryTokens } from "./src/token";
 import { RegistryAuthProtocolTokenPayload } from "./src/auth";
-import { registries } from "./src/registry/registry";
+import { ListRepositoriesResponse, registries } from "./src/registry/registry";
 import { RegistryHTTPClient } from "./src/registry/http";
 
 function createRequest(method: string, path: string, body: ReadableStream | null, headers = {}) {
@@ -112,6 +112,7 @@ describe("v2 manifests", () => {
       "content-type": "application/gzip",
       "docker-content-digest": sha256,
     });
+    await bindings.REGISTRY.delete(`${name}/manifests/${reference}`);
   });
 
   test("PUT /v2/:name/manifests/:reference works", () => createManifest("hello-world-main", "{}", "hello"));
@@ -371,3 +372,50 @@ describe("http client", () => {
     expect("exists" in res && res.exists).toBe(false);
   });
 });
+
+describe("push and catalog", () => {
+  test("push and then use the catalog", async () => {
+    await createManifest("hello-world-main", "{}", "hello");
+    await createManifest("hello-world-main", "{}", "latest");
+    await createManifest("hello-world-main", "{}", "hello-2");
+    await createManifest("hello", "{}", "hello");
+    await createManifest("hello/hello", "{}", "hello");
+
+    const response = await fetchUnauth(createRequest("GET", "/v2/_catalog", null));
+    expect(response.ok).toBeTruthy();
+    const body = (await response.json()) as { repositories: string[] };
+    expect(body).toEqual({
+      repositories: ["hello-world-main", "hello/hello", "hello"],
+    });
+    const expectedRepositories = body.repositories;
+    const tagsRes = await fetchUnauth(createRequest("GET", `/v2/hello-world-main/tags/list?n=1000`, null));
+    const tags = (await tagsRes.json()) as TagsList;
+    expect(tags.name).toEqual("hello-world-main");
+    expect(tags.tags).toEqual([
+      "hello",
+      "hello-2",
+      "latest",
+      "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
+    ]);
+
+    const repositoryBuildUp: string[] = [];
+    let currentPath = "/v2/_catalog?n=1";
+    for (let i = 0; i < 3; i++) {
+      const response = await fetchUnauth(createRequest("GET", currentPath, null));
+      expect(response.ok).toBeTruthy();
+      const body = (await response.json()) as { repositories: string[] };
+      if (body.repositories.length === 0) {
+        break;
+      }
+      expect(body.repositories).toHaveLength(1);
+
+      repositoryBuildUp.push(...body.repositories);
+      const url = new URL(response.headers.get("Link")!.split(";")[0].trim());
+      console.log(url.pathname, url.search, response.headers.get("Link"));
+      currentPath = url.pathname + url.search;
+      console.log(currentPath);
+    }
+
+    expect(repositoryBuildUp).toEqual(expectedRepositories);
+  });
+});

src/errors.ts

@@ -24,14 +24,28 @@ export class AuthErrorResponse extends Response {
 
 export class RangeError extends Response {
   constructor(stateStr: string, state: State) {
-    super("{}", {
-      status: 416,
-      headers: {
-        "Location": `/v2/${state.name}/blobs/uploads/${state.registryUploadId}?_state=${stateStr}`,
-        "Range": `0-${state.byteRange - 1}`,
-        "Docker-Upload-UUID": state.registryUploadId,
+    super(
+      JSON.stringify({
+        errors: [
+          {
+            code: "RANGE_ERROR",
+            message: `state ${stateStr} is not satisfiable (upload id: ${state.registryUploadId})`,
+            detail: {
+              ...state,
+              string: stateStr,
+            },
+          },
+        ],
+      }),
+      {
+        status: 416,
+        headers: {
+          "Location": `/v2/${state.name}/blobs/uploads/${state.registryUploadId}?_state=${stateStr}`,
+          "Range": `0-${state.byteRange - 1}`,
+          "Docker-Upload-UUID": state.registryUploadId,
+        },
       },
-    });
+    );
   }
 }
 

src/registry/http.ts

@@ -7,6 +7,7 @@ import {
   FinishedUploadObject,
   GetLayerResponse,
   GetManifestResponse,
+  ListRepositoriesResponse,
   PutManifestResponse,
   Registry,
   RegistryConfiguration,
@@ -64,7 +65,6 @@ function ctxIntoRequest(ctx: HTTPContext, url: URL, method: string, path: string
   const urlReq = `${url.protocol}//${ctx.authContext.service}/v2${
     ctx.repository === "" ? "/" : ctx.repository + "/"
   }${path}`;
-  console.log("Doing request:", urlReq);
   return new Request(urlReq, {
     method,
     body,
@@ -467,6 +467,10 @@ export class RegistryHTTPClient implements Registry {
   ): Promise<RegistryError | FinishedUploadObject> {
     throw new Error("unimplemented");
   }
+
+  async listRepositories(_limit?: number, _last?: string): Promise<RegistryError | ListRepositoriesResponse> {
+    throw new Error("unimplemented");
+  }
 }
 
 // AuthType defined the supported auth types

src/registry/r2.ts

@@ -132,30 +132,97 @@ export class R2Registry implements Registry {
   }
 
   async listRepositories(limit?: number, last?: string): Promise<RegistryError | ListRepositoriesResponse> {
-    const env = this.env;
+    // The idea in listRepositories is list all entries in the R2 bucket and map them to repositories.
+    // We do this by taking advantage of the name format in the R2 bucket:
+    // name format is:
+    //   <path>/<'blobs' | 'manifests'>/<name>
+    // This means we slice the last two items in the key and add them to our hash map.
+    // At the end, we start skipping entries until we find another unique key, then we return that entry as startAfter.
+
     const options = {
-      limit: limit ? limit : 1000,
-      delimiter: "/",
-      startAfter: last,
-    }
-    const r2Objects = (await env.REGISTRY.list(options));
+      limit: limit ?? 1000,
+      startAfter: last ?? undefined,
+    };
+    const repositories: Record<string, {}> = {};
+    let totalRecords = 0;
+    let lastSeen: string | undefined;
+    const objectExistsInPath = (entry: string) => {
+      const parts = entry.split("/");
+      const repository = parts.slice(0, parts.length - 2).join("/");
+      return repository in repositories;
+    };
 
-    let truncated = r2Objects.truncated;
-    let cursor = truncated ? r2Objects.cursor : undefined;
+    const addObjectPath = (object: R2Object) => {
+      // update lastSeen for cursoring purposes
+      lastSeen = object.key;
+      // don't add if seen before
+      if (totalRecords >= options.limit) return;
+      // skip either 'manifests' or 'blobs'
+      // name format is:
+      // <path>/<'blobs' | 'manifests'>/<name>
+      const parts = object.key.split("/");
+      const repository = parts.slice(0, parts.length - 2).join("/");
+      if (!(repository in repositories)) {
+        totalRecords++;
+      }
 
-    while (truncated) {
-      const next = await env.REGISTRY.list({
-        ...options,
-        cursor: cursor,
+      repositories[repository] = {};
+    };
+
+    const r2Objects = await this.env.REGISTRY.list({
+      limit: options.limit,
+      startAfter: options.startAfter,
+    });
+    r2Objects.objects.forEach((path) => addObjectPath(path));
+    let cursor = r2Objects.truncated ? r2Objects.cursor : undefined;
+    while (cursor !== undefined && totalRecords < options.limit) {
+      const next = await this.env.REGISTRY.list({
+        limit: options.limit,
+        cursor,
       });
-      r2Objects.objects.push(...next.objects);
-
-      truncated = next.truncated;
-      cursor = next.cursor
+      next.objects.forEach((path) => addObjectPath(path));
+      if (next.truncated) {
+        cursor = next.cursor;
+      } else {
+        cursor = undefined;
+      }
+    }
+
+    while (cursor !== undefined && typeof lastSeen === "string" && objectExistsInPath(lastSeen)) {
+      const nextList: R2Objects = await this.env.REGISTRY.list({
+        limit: 1000,
+        cursor,
+      });
+
+      let found = false;
+      // Search for the next object in the list
+      for (const object of nextList.objects) {
+        lastSeen = object.key;
+        if (!objectExistsInPath(lastSeen)) {
+          found = true;
+          break;
+        }
+      }
+
+      if (found) break;
+
+      if (nextList.truncated) {
+        // jump to the next list and try to find a
+        // repository that hasn't been returned in this response
+        cursor = nextList.cursor;
+      } else {
+        // we arrived to the end of the list, no more cursor
+        cursor = undefined;
+      }
+    }
+
+    if (cursor === undefined) {
+      lastSeen = undefined;
     }
 
     return {
-      repositories: r2Objects.delimitedPrefixes.map((name)=> name.endsWith('/') ? name.slice(0, -1) : name)
+      repositories: Object.keys(repositories),
+      cursor: lastSeen,
     };
   }
 

src/registry/registry.ts

@@ -43,11 +43,10 @@ export type CheckManifestResponse =
       exists: false;
     };
 
-export type ListRepositoriesResponse =
-  {
-    repositories: string[];
-  }
-
+export type ListRepositoriesResponse = {
+  repositories: string[];
+  cursor?: string;
+};
 
 // Response layerExists call
 export type CheckLayerResponse =

src/router.ts

@@ -27,15 +27,27 @@ v2Router.get("/", async (_req, _env: Env) => {
 
 v2Router.get("/_catalog", async (req, env: Env) => {
   const { n, last } = req.query;
-  const res = await env.REGISTRY_CLIENT.listRepositories(
+  const response = await env.REGISTRY_CLIENT.listRepositories(
     n ? parseInt(n?.toString()) : undefined,
-    last?.toString()
+    last?.toString(),
   );
+  if ("response" in response) {
+    return response.response;
+  }
 
-  return new Response(JSON.stringify(res));
+  const url = new URL(req.url);
+  return new Response(
+    JSON.stringify({
+      repositories: response.repositories,
+    }),
+    {
+      headers: {
+        Link: `${url.protocol}//${url.hostname}${url.pathname}?n=${n ?? 1000}&last=${response.cursor ?? ""}; rel=next`,
+      },
+    },
+  );
 });
 
-
 v2Router.delete("/:name+/manifests/:reference", async (req, env: Env) => {
   // deleting a manifest works by retrieving the """main""" manifest that its key is a sha,
   // and then going through every tag and removing it
@@ -49,17 +61,17 @@ v2Router.delete("/:name+/manifests/:reference", async (req, env: Env) => {
   //
   // If somehow we need to remove by paginating, we accept a last query param
 
-  const { last } = req.query;
+  const { last, limit } = req.query;
   const { name, reference } = req.params;
   // Reference is ALWAYS a sha256
   const manifest = await env.REGISTRY.head(`${name}/manifests/${reference}`);
   if (manifest === null) {
     return new Response(JSON.stringify(ManifestUnknownError), { status: 404 });
   }
-
+  const limitInt = parseInt(limit?.toString() ?? "1000", 10);
   const tags = await env.REGISTRY.list({
     prefix: `${name}/manifests`,
-    limit: 1000,
+    limit: isNaN(limitInt) ? 1000 : limitInt,
     startAfter: last?.toString(),
   });
   for (const tag of tags.objects) {
@@ -76,7 +88,7 @@ v2Router.delete("/:name+/manifests/:reference", async (req, env: Env) => {
     return new Response(JSON.stringify(ManifestTagsListTooBigError), {
       status: 400,
       headers: {
-        "Link": `${req.url}/last=${tags.objects[tags.objects.length - 1]}; rel=next`,
+        "Link": `${req.url}/?last=${tags.truncated ? tags.cursor : ""}; rel=next`,
         "Content-Type": "application/json",
       },
     });