feat!: add delta base to Bucket prefix

Delta encoding for Entry seq field
The base of a child bucket will be the seq of the boundary

Closes #8

src/codec.ts

@@ -1,14 +1,17 @@
 import { decode, encode } from "@ipld/dag-cbor";
 import { sha256 } from "@noble/hashes/sha256";
 import type { ByteView } from "multiformats";
+import { compareEntries } from "./compare.js";
 import { DefaultBucket, DefaultEntry } from "./impls.js";
 import { Bucket, Entry, Prefix } from "./interface.js";
+import { entriesToDeltaBase } from "./utils.js";
 
 type EncodedEntry = [Entry["seq"], Entry["key"], Entry["val"]];
 
 export interface EncodedBucket {
   level: number;
   average: number;
+  base: number;
   entries: EncodedEntry[];
 }
 
@@ -39,7 +42,7 @@ const getValidatedPrefix = (prefix: unknown): Prefix => {
     throw new TypeError("Expected bucket prefix to be an object.");
   }
 
-  const { average, level } = prefix as Partial<Prefix>;
+  const { average, level, base } = prefix as Partial<Prefix>;
 
   if (typeof average !== "number") {
     throw new TypeError("Expected prefix average field to be a number.");
@@ -49,34 +52,53 @@ const getValidatedPrefix = (prefix: unknown): Prefix => {
     throw new TypeError("Expected prefix level field to be a number.");
   }
 
-  return { average, level };
+  if (typeof base !== "number") {
+    throw new TypeError("Expected prefix base field to be a number.");
+  }
+
+  return { average, level, base };
 };
 
 const getValidatedBucket = (bucket: unknown): EncodedBucket => {
   if (typeof bucket !== "object" || bucket == null) {
     throw new TypeError("Expected bucket to be an object.");
   }
 
-  const { average, level } = getValidatedPrefix(bucket);
+  const { average, level, base } = getValidatedPrefix(bucket);
 
   const { entries } = bucket as Partial<EncodedBucket>;
 
   if (typeof entries !== "object" || !Array.isArray(entries)) {
     throw new TypeError("Expected bucket entries field to be a number.");
   }
 
-  return { average, level, entries };
+  return { average, level, base, entries };
 };
 
 export function encodeBucket(
   average: number,
   level: number,
   entries: Entry[],
 ): ByteView<EncodedBucket> {
+  const base = entriesToDeltaBase(entries);
+
+  entries.sort(compareEntries);
+
+  let i = entries.length;
+  const encodedEntries: EncodedEntry[] = new Array(i);
+  let delta = base;
+  while (i > 0) {
+    i--;
+    const { seq, key, val } = entries[i]!;
+    encodedEntries[i] = [delta - seq, key, val];
+    delta = seq;
+  }
+
   return encode({
     average,
     level,
-    entries: entries.map(({ seq, key, val }) => [seq, key, val]),
+    base,
+    entries: encodedEntries,
   });
 }
 
@@ -89,6 +111,7 @@ export function decodeBucket(
   const {
     average,
     level,
+    base,
     entries: encodedEntries,
   } = getValidatedBucket(decoded);
 
@@ -104,12 +127,21 @@ export function decodeBucket(
     );
   }
 
+  if (base !== expectedPrefix.base) {
+    throw new TypeError(
+      `Expect prefix to have base ${expectedPrefix.base}. Received prefix with base ${base}`,
+    );
+  }
+
   // could validate boundaries and tuple order here
-  let i = 0;
-  const entries: Entry[] = new Array(encodedEntries.length);
-  for (const entry of encodedEntries) {
-    entries[i] = new DefaultEntry(...getValidatedEntry(entry));
-    i++;
+  let i = encodedEntries.length;
+  const entries: Entry[] = new Array(i);
+  let delta: number = base;
+  while (i > 0) {
+    i--;
+    const encodedEntry = encodedEntries[i];
+    const [seq, key, val] = getValidatedEntry(encodedEntry);
+    entries[i] = new DefaultEntry((delta -= seq), key, val);
   }
 
   return new DefaultBucket(average, level, entries, bytes, sha256(bytes));

src/cursor.ts

@@ -310,6 +310,7 @@ const moveToLevel = async (
       const bucket = await loadBucket(state.blockstore, digest, {
         ...bucketToPrefix(bucketOf(state)),
         level: levelOf(state) - 1,
+        base: entryOf(state).seq,
       });
 
       if (bucket.entries.length === 0) {

src/impls.ts

@@ -1,7 +1,7 @@
 import { base32 } from "multiformats/bases/base32";
 import { CID } from "multiformats/cid";
 import { Bucket, Entry, ProllyTree } from "./interface.js";
-import { bucketDigestToCid } from "./utils.js";
+import { bucketDigestToCid, entriesToDeltaBase } from "./utils.js";
 
 const nodeInspectSymbol = Symbol.for("entryjs.util.inspect.custom");
 
@@ -28,6 +28,7 @@ export class DefaultEntry implements Entry {
 export class DefaultBucket implements Bucket {
   #bytes: Uint8Array;
   #digest: Uint8Array;
+  readonly base: number;
 
   constructor(
     readonly average: number,
@@ -38,6 +39,7 @@ export class DefaultBucket implements Bucket {
   ) {
     this.#bytes = bytes;
     this.#digest = digest;
+    this.base = entriesToDeltaBase(entries);
   }
 
   getBytes(): Uint8Array {

src/interface.ts

@@ -12,6 +12,7 @@ export interface Entry extends Tuple {
 export interface Prefix {
   readonly average: number; // same for all buckets of the same tree
   readonly level: number; // changes based on level of the bucket in the tree, leaves are always level 0
+  readonly base: number; // base number for delta encoding of entry seq field
 }
 
 export interface Bucket extends Prefix {

src/utils.ts

@@ -1,5 +1,6 @@
 import { code as cborCode } from "@ipld/dag-cbor";
 import { sha256 } from "@noble/hashes/sha256";
+import { lastElement } from "@tabcat/ith-element";
 import { Blockstore } from "interface-blockstore";
 import { CID } from "multiformats/cid";
 import { create as createMultihashDigest } from "multiformats/hashes/digest";
@@ -39,11 +40,15 @@ export const entryToTuple = ({ seq, key }: Tuple): Tuple => ({
  * @param prefix
  * @returns
  */
-export const bucketToPrefix = ({ average, level }: Prefix): Prefix => ({
+export const bucketToPrefix = ({ average, level, base }: Prefix): Prefix => ({
   average,
   level,
+  base,
 });
 
+export const entriesToDeltaBase = (entries: Entry[]): number =>
+  entries.length > 0 ? lastElement(entries).seq : 0;
+
 /**
  * Creates a new bucket from the provided entries. Does not handle boundary creation.
  * This is a low level function and is easy to use incorrectly.

test/codec.test.ts

@@ -1,7 +1,7 @@
 import { encode } from "@ipld/dag-cbor";
 import { describe, expect, it } from "vitest";
 import { decodeBucket, encodeBucket } from "../src/codec.js";
-import { emptyBucket, encodedEmptyBucket } from "./helpers/constants.js";
+import { base, emptyBucket, encodedEmptyBucket } from "./helpers/constants.js";
 
 const { average, level, entries } = emptyBucket;
 
@@ -12,77 +12,101 @@ describe("codec", () => {
         encodedEmptyBucket,
       );
     });
+
+    it("delta encodes the seq fields of entries", () => {});
   });
 
   describe("decodeBucket", () => {
     it("decodes a bucket", () => {
       expect(
-        decodeBucket(encodedEmptyBucket, { average, level }),
+        decodeBucket(encodedEmptyBucket, { average, level, base }),
       ).to.deep.equal(emptyBucket);
     });
 
+    it("delta decodes the seq fields entries", () => {});
+
     it("throws when expected average does not match", () => {
       expect(() =>
-        decodeBucket(encodedEmptyBucket, { average: -1, level }),
+        decodeBucket(encodedEmptyBucket, { average: -1, level, base }),
       ).toThrow();
     });
 
     it("throws when expected level does not match", () => {
       expect(() =>
-        decodeBucket(encodedEmptyBucket, { average, level: -1 }),
+        decodeBucket(encodedEmptyBucket, { average, level: -1, base }),
       ).toThrow();
     });
 
     it("throws when decoded bucket is not an object", () => {
-      expect(() => decodeBucket(encode(null), { average, level })).toThrow(
-        "Expected bucket to be an object.",
-      );
+      expect(() =>
+        decodeBucket(encode(null), { average, level, base }),
+      ).toThrow("Expected bucket to be an object.");
     });
 
     it("throws when decoded average is not a number", () => {
-      expect(() => decodeBucket(encode({}), { average, level })).toThrow(
+      expect(() => decodeBucket(encode({}), { average, level, base })).toThrow(
         "Expected prefix average field to be a number.",
       );
     });
 
     it("throws when decoded level is not a number", () => {
       expect(() =>
-        decodeBucket(encode({ average }), { average, level }),
+        decodeBucket(encode({ average }), { average, level, base }),
       ).toThrow("Expected prefix level field to be a number.");
     });
 
+    it("throws when decoded base is not a number", () => {
+      expect(() =>
+        decodeBucket(encode({ average, level }), { average, level, base }),
+      ).toThrow("Expected prefix base field to be a number.");
+    });
+
     it("throws when decoded entries is not an array", () => {
       expect(() =>
-        decodeBucket(encode({ average, level }), { average, level }),
+        decodeBucket(encode({ average, level, base }), {
+          average,
+          level,
+          base,
+        }),
       ).toThrow("Expected bucket entries field to be a number.");
     });
 
     it("throws when decoded entries contains an invalid seq", () => {
       expect(() =>
         decodeBucket(
-          encode({ average, level, entries: [[null, null, null]] }),
+          encode({ average, level, base, entries: [[null, null, null]] }),
           {
             average,
             level,
+            base,
           },
         ),
       ).toThrow("Expected entry seq field to be a number.");
     });
 
     it("throws when decoded entries contains an invalid hash", () => {
       expect(() =>
-        decodeBucket(encode({ average, level, entries: [[0, null, null]] }), {
-          average,
-          level,
-        }),
+        decodeBucket(
+          encode({ average, level, base, entries: [[0, null, null]] }),
+          {
+            average,
+            level,
+            base,
+          },
+        ),
       ).toThrow("Expected entry key field to be a byte array.");
     });
 
     it("throws when decoded entries contains an invalid val", () => {
       expect(() =>
         decodeBucket(
-          encode({ average, level, entries: [[0, new Uint8Array(), null]] }),
-          { average, level },
+          encode({
+            average,
+            level,
+            base,
+            entries: [[0, new Uint8Array(), null]],
+          }),
+          { average, level, base },
         ),
       ).toThrow("Expected entry val field to be a byte array.");
     });

test/helpers/constants.ts

@@ -23,11 +23,13 @@ export const entry = new DefaultEntry(seq, key, val);
 
 export const average = 32;
 export const level = 0;
-export const prefix = { average, level };
+export const base = 0;
+export const prefix = { average, level, base };
 export const entries = [entry];
 export const encodedBucket = encode({
   average,
   level,
+  base,
   entries: [[seq, key, val]],
 });
 export const bucketDigest = sha256(encodedBucket);
@@ -38,7 +40,12 @@ export const bucket = new DefaultBucket(
   encodedBucket,
   bucketDigest,
 );
-export const encodedEmptyBucket = encode({ average, level, entries: [] });
+export const encodedEmptyBucket = encode({
+  average,
+  level,
+  base: 0,
+  entries: [],
+});
 export const emptyBucket = new DefaultBucket(
   average,
   level,