diff --git a/.github/ISSUE_TEMPLATE/package--ethereumjs-verkle.md b/.github/ISSUE_TEMPLATE/package--ethereumjs-verkle.md new file mode 100644 index 0000000000..fbe58a562d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/package--ethereumjs-verkle.md @@ -0,0 +1,7 @@ +--- +name: 'Package: @ethereumjs/verkle' +about: Create issue for @ethereumjs/verkle package +title: '' +labels: 'package: verkle' +assignees: '' +--- diff --git a/.github/workflows/verkle-build.yml b/.github/workflows/verkle-build.yml new file mode 100644 index 0000000000..5517c34353 --- /dev/null +++ b/.github/workflows/verkle-build.yml @@ -0,0 +1,42 @@ +name: Verkle +on: + push: + branches: [master, develop] + tags: ['*'] + pull_request: + types: [opened, reopened, synchronize] + workflow_dispatch: + +env: + cwd: ${{github.workspace}}/packages/verkle + +defaults: + run: + working-directory: packages/verkle + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test-verkle: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18] + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - run: npm ci --omit=peer + working-directory: ${{github.workspace}} + + - run: npm run lint + - run: npm run test:node # Only run node tests for now until vitest browser test issues are sorted out diff --git a/README.md b/README.md index 9e838ae38c..8d51866e2b 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Below you can find a list of the packages included in this repository. | [@ethereumjs/trie][trie-package] | [![NPM Package][trie-npm-badge]][trie-npm-link] | [![Trie Issues][trie-issues-badge]][trie-issues-link] | [![Actions Status][trie-actions-badge]][trie-actions-link] | [![Code Coverage][trie-coverage-badge]][trie-coverage-link] | | [@ethereumjs/tx][tx-package] | [![NPM Package][tx-npm-badge]][tx-npm-link] | [![Tx Issues][tx-issues-badge]][tx-issues-link] | [![Actions Status][tx-actions-badge]][tx-actions-link] | [![Code Coverage][tx-coverage-badge]][tx-coverage-link] | | [@ethereumjs/util][util-package] | [![NPM Package][util-npm-badge]][util-npm-link] | [![Util Issues][util-issues-badge]][util-issues-link] | [![Actions Status][util-actions-badge]][util-actions-link] | [![Code Coverage][util-coverage-badge]][util-coverage-link] | +| [@ethereumjs/verkle][verkle-package] | [![NPM Package][verkle-npm-badge]][verkle-npm-link] | [![VM Issues][verkle-issues-badge]][verkle-issues-link] | [![Actions Status][verkle-actions-badge]][verkle-actions-link] | [![Code Coverage][verkle-coverage-badge]][verkle-coverage-link] | | [@ethereumjs/vm][vm-package] | [![NPM Package][vm-npm-badge]][vm-npm-link] | [![VM Issues][vm-issues-badge]][vm-issues-link] | [![Actions Status][vm-actions-badge]][vm-actions-link] | [![Code Coverage][vm-coverage-badge]][vm-coverage-link] | | [@ethereumjs/wallet][wallet-package] | [![NPM Package][wallet-npm-badge]][wallet-npm-link] | [![StateManager Issues][wallet-issues-badge]][wallet-issues-link] | [![Actions Status][wallet-actions-badge]][wallet-actions-link] | [![Code Coverage][wallet-coverage-badge]][wallet-coverage-link] | @@ -235,6 +236,15 @@ Most packages are [MPL-2.0](=18" + } + }, + "packages/verkle/node_modules/lru-cache": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.0.tgz", + "integrity": "sha512-svTf/fzsKHffP42sujkO/Rjs37BCIsQVRCeNYIm9WN8rgT7ffoUnRtZCqU+6BqcSBdv8gwJeTz8knJpgACeQMw==", + "engines": { + "node": "14 || >=16.14" + } + }, "packages/vm": { "name": "@ethereumjs/vm", "version": "7.0.0", diff --git a/packages/util/src/bytes.ts b/packages/util/src/bytes.ts index f573fea903..b4a5536011 100644 --- a/packages/util/src/bytes.ts +++ b/packages/util/src/bytes.ts @@ -444,5 +444,61 @@ export const concatBytes = (...arrays: Uint8Array[]): Uint8Array => { return result } +/** + * @notice Convert a Uint8Array to a 32-bit integer + * @param {Uint8Array} bytes The input Uint8Array from which to read the 32-bit integer. + * @param {boolean} littleEndian True for little-endian, undefined or false for big-endian. + * @throws {Error} If the input Uint8Array has a length less than 4. + * @return {number} The 32-bit integer read from the input Uint8Array. + */ +export function bytesToInt32(bytes: Uint8Array, littleEndian: boolean = false): number { + if (bytes.length < 4) { + throw new Error('The input Uint8Array must have at least 4 bytes.') + } + const dataView = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + return dataView.getInt32(0, littleEndian) +} + +/** + * @notice Convert a Uint8Array to a 64-bit bigint + * @param {Uint8Array} bytes The input Uint8Array from which to read the 64-bit bigint. + * @param {boolean} littleEndian True for little-endian, undefined or false for big-endian. + * @throws {Error} If the input Uint8Array has a length less than 8. + * @return {bigint} The 64-bit bigint read from the input Uint8Array. + */ +export function bytesToBigInt64(bytes: Uint8Array, littleEndian: boolean = false): bigint { + if (bytes.length < 8) { + throw new Error('The input Uint8Array must have at least 8 bytes.') + } + const dataView = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + return dataView.getBigInt64(0, littleEndian) +} + +/** + * @notice Convert a 32-bit integer to a Uint8Array. + * @param {number} value The 32-bit integer to convert. + * @param {boolean} littleEndian True for little-endian, undefined or false for big-endian. + * @return {Uint8Array} A Uint8Array of length 4 containing the integer. + */ +export function int32ToBytes(value: number, littleEndian: boolean = false): Uint8Array { + const buffer = new ArrayBuffer(4) + const dataView = new DataView(buffer) + dataView.setInt32(0, value, littleEndian) + return new Uint8Array(buffer) +} + +/** + * @notice Convert a 64-bit bigint to a Uint8Array. + * @param {bigint} value The 64-bit bigint to convert. + * @param {boolean} littleEndian True for little-endian, undefined or false for big-endian. + * @return {Uint8Array} A Uint8Array of length 8 containing the bigint. + */ +export function bigInt64ToBytes(value: bigint, littleEndian: boolean = false): Uint8Array { + const buffer = new ArrayBuffer(8) + const dataView = new DataView(buffer) + dataView.setBigInt64(0, value, littleEndian) + return new Uint8Array(buffer) +} + // eslint-disable-next-line no-restricted-imports export { bytesToUtf8, equalsBytes, utf8ToBytes } from 'ethereum-cryptography/utils.js' diff --git a/packages/verkle/.c8rc.json b/packages/verkle/.c8rc.json new file mode 100644 index 0000000000..52eb43c23b --- /dev/null +++ b/packages/verkle/.c8rc.json @@ -0,0 +1,4 @@ +{ + "extends": "../../config/.c8rc.json", + "include": ["src/**/*.ts"] +} diff --git a/packages/verkle/.eslintrc.cjs b/packages/verkle/.eslintrc.cjs new file mode 100644 index 0000000000..884b3d6ebe --- /dev/null +++ b/packages/verkle/.eslintrc.cjs @@ -0,0 +1,14 @@ +module.exports = { + extends: '../../config/eslint.cjs', + parserOptions: { + project: ['./tsconfig.json', './tsconfig.benchmarks.json'], + }, + overrides: [ + { + files: ['benchmarks/*.ts'], + rules: { + 'no-console': 'off', + }, + }, + ], +} diff --git a/packages/verkle/.gitignore b/packages/verkle/.gitignore new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/verkle/.gitignore @@ -0,0 +1 @@ + diff --git a/packages/verkle/.npmignore b/packages/verkle/.npmignore new file mode 100644 index 0000000000..55c65cf8bd --- /dev/null +++ b/packages/verkle/.npmignore @@ -0,0 +1,2 @@ +test/ +src/ \ No newline at end of file diff --git a/packages/verkle/.prettierignore b/packages/verkle/.prettierignore new file mode 100644 index 0000000000..9fa0fb74e1 --- /dev/null +++ b/packages/verkle/.prettierignore @@ -0,0 +1,6 @@ +node_modules +.vscode +dist +.nyc_output +*.json +docs \ No newline at end of file diff --git a/packages/verkle/CHANGELOG.md b/packages/verkle/CHANGELOG.md new file mode 100644 index 0000000000..459edbc1b3 --- /dev/null +++ b/packages/verkle/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +(modification: no type change headlines) and this project adheres to +[Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [0.0.1] - 2023-10-15 + +- Initial development release diff --git a/packages/verkle/README.md b/packages/verkle/README.md new file mode 100644 index 0000000000..d5440ff799 --- /dev/null +++ b/packages/verkle/README.md @@ -0,0 +1,112 @@ +# @ethereumjs/verkle + +[![NPM Package][verkle-npm-badge]][verkle-npm-link] +[![GitHub Issues][verkle-issues-badge]][verkle-issues-link] +[![Actions Status][verkle-actions-badge]][verkle-actions-link] +[![Code Coverage][verkle-coverage-badge]][verkle-coverage-link] +[![Discord][discord-badge]][discord-link] + +| Implementation of [Verkle Trees](https://ethereum.org/en/roadmap/verkle-trees/) as specified in [EIP-6800](https://eips.ethereum.org/EIPS/eip-6800) | +| --------------------------------------------------------------------------------------------------------------------------------------------------- | + +> Verkle trees are a cryptographic data structure proposed for use in Ethereum to optimize storage and transaction verification. They combine features of Merkle Patricia Tries and Vector Commitment Trees to offer efficient data verification with smaller proof sizes. The goal is to improve scalability and efficiency in Ethereum's network operations. + +This package is currently in early alpha and is a work in progress. It is not intended for use in production environments, but rather for research and development purposes. Any help in improving the package is very much welcome. + +## Installation + +To obtain the latest version, simply require the project using `npm`: + +```shell +npm install @ethereumjs/verkle +``` + +## Usage + +This class implements the basic [Modified Merkle Patricia Trie](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/) in the `Trie` base class, which you can use with the `useKeyHashing` option set to `true` to create a trie which stores values under the `keccak256` hash of its keys (this is the Trie flavor which is used in Ethereum production systems). + +Checkpointing functionality to `Trie` through the methods `checkpoint`, `commit` and `revert`. + +It is best to select the variant that is most appropriate for your unique use case. + +### Initialization and Basic Usage + +```typescript +import { VerkleTrie } from '@ethereumjs/verkle' +import { bytesToUtf8, utf8ToBytes } from 'ethereumjs/util' + +const trie = new VerkleTrie() + +async function test() { + await trie.put(utf8ToBytes('test'), utf8ToBytes('one')) + const value = await trie.get(utf8ToBytes('test')) + console.log(value ? bytesToUtf8(value) : 'not found') // 'one' +} + +test() +``` + +## Proofs + +### Verkle Proofs + +The EthereumJS Verkle package is still in its infancy, and as such, it does not currently support Verkle proof creation and verification. Support for Verkle proofs will be added eventually. + +## Examples + +You can find additional examples complete with detailed explanations [here](./examples/README.md). + +## Browser + +With the breaking release round in Summer 2023 we have added hybrid ESM/CJS builds for all our libraries (see section below) and have eliminated many of the caveats which had previously prevented frictionless browser usage. + +It is now easily possible to run a browser build of one of the EthereumJS libraries within a modern browser using the provided ESM build. For a setup example see [./examples/browser.html](./examples/browser.html). + +## API + +### Docs + +Generated TypeDoc API [Documentation](./docs/README.md) + +### Hybrid CJS/ESM Builds + +With the breaking releases from Summer 2023 we have started to ship our libraries with both CommonJS (`cjs` folder) and ESM builds (`esm` folder), see `package.json` for the detailed setup. + +If you use an ES6-style `import` in your code, files from the ESM build will be used: + +```typescript +import { EthereumJSClass } from '@ethereumjs/[PACKAGE_NAME]' +``` + +If you use Node.js-specific `require`, the CJS build will be used: + +```typescript +const { EthereumJSClass } = require('@ethereumjs/[PACKAGE_NAME]') +``` + +Using ESM will give you additional advantages over CJS beyond browser usage like static code analysis / Tree Shaking, which CJS cannot provide. + +## References + +- Wiki + - [Overview of verkle tries](https://ethereum.org/en/roadmap/verkle-trees/) + - [Verkle tries General Resource](https://verkle.info/) + +## EthereumJS + +See our organizational [documentation](https://ethereumjs.readthedocs.io) for an introduction to `EthereumJS` as well as information on current standards and best practices. If you want to join for work or carry out improvements on the libraries, please review our [contribution guidelines](https://ethereumjs.readthedocs.io/en/latest/contributing.html) first. + +## License + +[MPL-2.0]() + +[discord-badge]: https://img.shields.io/static/v1?logo=discord&label=discord&message=Join&color=blue +[discord-link]: https://discord.gg/TNwARpR +[verkle-npm-badge]: https://img.shields.io/npm/v/@ethereumjs/verkle.svg +[verkle-npm-link]: https://www.npmjs.com/package/@ethereumjs/verkle +[verkle-issues-badge]: https://img.shields.io/github/issues/ethereumjs/ethereumjs-monorepo/package:%20verkle?label=issues +[verkle-issues-link]: https://github.com/ethereumjs/ethereumjs-monorepo/issues?q=is%3Aopen+is%3Aissue+label%3A"package%3A+verkle" +[verkle-actions-badge]: https://github.com/ethereumjs/ethereumjs-monorepo/workflows/Trie/badge.svg +[verkle-actions-link]: https://github.com/ethereumjs/ethereumjs-monorepo/actions?query=workflow%3A%22Trie%22 +[verkle-coverage-badge]: https://codecov.io/gh/ethereumjs/ethereumjs-monorepo/branch/master/graph/badge.svg?flag=verkle +[verkle-coverage-link]: https://codecov.io/gh/ethereumjs/ethereumjs-monorepo/tree/master/packages/verkle diff --git a/packages/verkle/package.json b/packages/verkle/package.json new file mode 100644 index 0000000000..ae2e074494 --- /dev/null +++ b/packages/verkle/package.json @@ -0,0 +1,60 @@ +{ + "name": "@ethereumjs/verkle", + "version": "0.0.1", + "description": "Implementation of verkle tries as used in Ethereum.", + "keywords": [ + "verkle", + "trie", + "ethereum" + ], + "homepage": "https://github.com/ethereumjs/ethereumjs-monorepo/tree/master/packages/verkle#readme", + "bugs": { + "url": "https://github.com/ethereumjs/ethereumjs-monorepo/issues?q=is%3Aissue+label%3A%22package%3A+verkle%22" + }, + "repository": { + "type": "git", + "url": "https://github.com/ethereumjs/ethereumjs-monorepo.git" + }, + "license": "MPL-2.0", + "author": "EthereumJS Team", + "contributors": [ + { + "name": "Gabriel Rocheleau", + "url": "https://github.com/gabrocheleau" + } + ], + "main": "dist/cjs/index.js", + "type": "commonjs", + "module": "dist/esm/index.js", + "exports": { + ".": { + "import": "./dist/esm/index.js", + "require": "./dist/cjs/index.js" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "../../config/cli/ts-build.sh", + "clean": "../../config/cli/clean-package.sh", + "coverage": "npx vitest run --coverage.enabled --coverage.reporter=lcov", + "docs:build": "typedoc --options typedoc.cjs", + "examples": "ts-node ../../scripts/examples-runner.ts -- verkle", + "lint": "../../config/cli/lint.sh", + "lint:diff": "../../config/cli/lint-diff.sh", + "lint:fix": "../../config/cli/lint-fix.sh", + "prepublishOnly": "../../config/cli/prepublish.sh", + "test": "npx vitest run", + "tsc": "../../config/cli/ts-compile.sh" + }, + "dependencies": { + "@ethereumjs/rlp": "5.0.0", + "@ethereumjs/util": "9.0.0", + "lru-cache": "^10.0.0" + }, + "engines": { + "node": ">=18" + } +} diff --git a/packages/verkle/src/db/checkpoint.ts b/packages/verkle/src/db/checkpoint.ts new file mode 100644 index 0000000000..6755796cc9 --- /dev/null +++ b/packages/verkle/src/db/checkpoint.ts @@ -0,0 +1,270 @@ +import { KeyEncoding, ValueEncoding, bytesToHex, hexToBytes } from '@ethereumjs/util' +import { LRUCache } from 'lru-cache' + +import type { Checkpoint, CheckpointDBOpts } from '../types.js' +import type { BatchDBOp, DB, DelBatch, PutBatch } from '@ethereumjs/util' + +/** + * DB is a thin wrapper around the underlying levelup db, + * which validates inputs and sets encoding type. + */ +export class CheckpointDB implements DB { + public checkpoints: Checkpoint[] + public db: DB + public readonly cacheSize: number + + // Starting with lru-cache v8 undefined and null are not allowed any more + // as cache values. At the same time our design works well, since undefined + // indicates for us that we know that the value is not present in the + // underlying trie database as well (so it carries real value). + // + // Solution here seems therefore adequate, other solutions would rather + // be some not so clean workaround. + // + // (note that @ts-ignore doesn't work since stripped on declaration (.d.ts) files) + protected _cache?: LRUCache + + _stats = { + cache: { + reads: 0, + hits: 0, + writes: 0, + }, + db: { + reads: 0, + hits: 0, + writes: 0, + }, + } + + /** + * Initialize a DB instance. + */ + constructor(opts: CheckpointDBOpts) { + this.db = opts.db + this.cacheSize = opts.cacheSize ?? 0 + // Roots of trie at the moment of checkpoint + this.checkpoints = [] + + if (this.cacheSize > 0) { + this._cache = new LRUCache({ + max: this.cacheSize, + updateAgeOnGet: true, + }) + } + } + + /** + * Flush the checkpoints and use the given checkpoints instead. + * @param {Checkpoint[]} checkpoints + */ + setCheckpoints(checkpoints: Checkpoint[]) { + this.checkpoints = [] + + for (let i = 0; i < checkpoints.length; i++) { + this.checkpoints.push({ + root: checkpoints[i].root, + keyValueMap: new Map(checkpoints[i].keyValueMap), + }) + } + } + + /** + * Is the DB during a checkpoint phase? + */ + hasCheckpoints() { + return this.checkpoints.length > 0 + } + + /** + * Adds a new checkpoint to the stack + * @param root + */ + checkpoint(root: Uint8Array) { + this.checkpoints.push({ keyValueMap: new Map(), root }) + } + + /** + * Commits the latest checkpoint + */ + async commit() { + const { keyValueMap } = this.checkpoints.pop()! + if (!this.hasCheckpoints()) { + // This was the final checkpoint, we should now commit and flush everything to disk + const batchOp: BatchDBOp[] = [] + for (const [key, value] of keyValueMap.entries()) { + if (value === undefined) { + batchOp.push({ + type: 'del', + key: hexToBytes(key), + opts: { + keyEncoding: KeyEncoding.Bytes, + }, + }) + } else { + batchOp.push({ + type: 'put', + key: hexToBytes(key), + value, + opts: { keyEncoding: KeyEncoding.Bytes, valueEncoding: ValueEncoding.Bytes }, + }) + } + } + await this.batch(batchOp) + } else { + // dump everything into the current (higher level) diff cache + const currentKeyValueMap = this.checkpoints[this.checkpoints.length - 1].keyValueMap + for (const [key, value] of keyValueMap.entries()) { + currentKeyValueMap.set(key, value) + } + } + } + + /** + * Reverts the latest checkpoint + */ + async revert() { + const { root } = this.checkpoints.pop()! + return root + } + + /** + * @inheritDoc + */ + async get(key: Uint8Array): Promise { + const keyHex = bytesToHex(key) + if (this._cache !== undefined) { + const value = this._cache.get(keyHex) + this._stats.cache.reads += 1 + if (value !== undefined) { + this._stats.cache.hits += 1 + return value + } + } + + // Lookup the value in our diff cache. We return the latest checkpointed value (which should be the value on disk) + for (let index = this.checkpoints.length - 1; index >= 0; index--) { + if (this.checkpoints[index].keyValueMap.has(keyHex)) { + return this.checkpoints[index].keyValueMap.get(keyHex) + } + } + // Nothing has been found in diff cache, look up from disk + const value = await this.db.get(key, { + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, + }) + this._stats.db.reads += 1 + if (value !== undefined) { + this._stats.db.hits += 1 + } + this._cache?.set(keyHex, value) + if (this.hasCheckpoints()) { + // Since we are a checkpoint, put this value in diff cache, + // so future `get` calls will not look the key up again from disk. + this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, value) + } + + return value + } + + /** + * @inheritDoc + */ + async put(key: Uint8Array, value: Uint8Array): Promise { + const keyHex = bytesToHex(key) + if (this.hasCheckpoints()) { + // put value in diff cache + this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, value) + } else { + await this.db.put(key, value, { + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, + }) + this._stats.db.writes += 1 + + if (this._cache !== undefined) { + this._cache.set(keyHex, value) + this._stats.cache.writes += 1 + } + } + } + + /** + * @inheritDoc + */ + async del(key: Uint8Array): Promise { + const keyHex = bytesToHex(key) + if (this.hasCheckpoints()) { + // delete the value in the current diff cache + this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, undefined) + } else { + // delete the value on disk + await this.db.del(key, { + keyEncoding: KeyEncoding.Bytes, + }) + this._stats.db.writes += 1 + + if (this._cache !== undefined) { + this._cache.set(keyHex, undefined) + this._stats.cache.writes += 1 + } + } + } + + /** + * @inheritDoc + */ + async batch(opStack: BatchDBOp[]): Promise { + if (this.hasCheckpoints()) { + for (const op of opStack) { + if (op.type === 'put') { + await this.put(op.key, op.value) + } else if (op.type === 'del') { + await this.del(op.key) + } + } + } else { + const convertedOps = opStack.map((op) => { + const convertedOp = { + key: op.key, + value: op.type === 'put' ? op.value : undefined, + type: op.type, + opts: op.opts, + } + if (op.type === 'put') return convertedOp as PutBatch + else return convertedOp as DelBatch + }) + await this.db.batch(convertedOps) + } + } + + stats(reset = true) { + const stats = { ...this._stats, size: this._cache?.size ?? 0 } + if (reset) { + this._stats = { + cache: { + reads: 0, + hits: 0, + writes: 0, + }, + db: { + reads: 0, + hits: 0, + writes: 0, + }, + } + } + return stats + } + + /** + * @inheritDoc + */ + shallowCopy(): CheckpointDB { + return new CheckpointDB({ db: this.db, cacheSize: this.cacheSize }) + } + + open() { + return Promise.resolve() + } +} diff --git a/packages/verkle/src/db/index.ts b/packages/verkle/src/db/index.ts new file mode 100644 index 0000000000..63e8f6b033 --- /dev/null +++ b/packages/verkle/src/db/index.ts @@ -0,0 +1 @@ +export * from './checkpoint.js' diff --git a/packages/verkle/src/index.ts b/packages/verkle/src/index.ts new file mode 100644 index 0000000000..1965d0f02f --- /dev/null +++ b/packages/verkle/src/index.ts @@ -0,0 +1,2 @@ +export * from './types.js' +export * from './verkleTrie.js' diff --git a/packages/verkle/src/node/baseVerkleNode.ts b/packages/verkle/src/node/baseVerkleNode.ts new file mode 100644 index 0000000000..01aee3ebfe --- /dev/null +++ b/packages/verkle/src/node/baseVerkleNode.ts @@ -0,0 +1,33 @@ +import { RLP } from '@ethereumjs/rlp' + +import { type VerkleNodeInterface, type VerkleNodeOptions, type VerkleNodeType } from './types.js' + +import type { Point } from '../types.js' + +export abstract class BaseVerkleNode implements VerkleNodeInterface { + public commitment: Point + public depth: number + + constructor(options: VerkleNodeOptions[T]) { + this.commitment = options.commitment + this.depth = options.depth + } + + abstract commit(): Point + + // Hash returns the field representation of the commitment. + hash(): Uint8Array { + throw new Error('Not implemented') + } + + abstract insert(key: Uint8Array, value: Uint8Array, nodeResolverFn: () => void): void + + abstract raw(): Uint8Array[] + + /** + * @returns the RLP serialized node + */ + serialize(): Uint8Array { + return RLP.encode(this.raw()) + } +} diff --git a/packages/verkle/src/node/index.ts b/packages/verkle/src/node/index.ts new file mode 100644 index 0000000000..c786638cde --- /dev/null +++ b/packages/verkle/src/node/index.ts @@ -0,0 +1,4 @@ +export * from './baseVerkleNode.js' +export * from './internalNode.js' +export * from './leafNode.js' +export * from './types.js' diff --git a/packages/verkle/src/node/internalNode.ts b/packages/verkle/src/node/internalNode.ts new file mode 100644 index 0000000000..9371744124 --- /dev/null +++ b/packages/verkle/src/node/internalNode.ts @@ -0,0 +1,120 @@ +import { equalsBytes } from '@ethereumjs/util' + +import { POINT_IDENTITY } from '../util/crypto.js' + +import { BaseVerkleNode } from './baseVerkleNode.js' +import { LeafNode } from './leafNode.js' +import { NODE_WIDTH, VerkleNodeType } from './types.js' + +import type { Point } from '../types.js' +import type { VerkleNode, VerkleNodeOptions } from './types.js' + +export class InternalNode extends BaseVerkleNode { + // Array of references to children nodes + public children: Array + public copyOnWrite: Record + public type = VerkleNodeType.Internal + + /* TODO: options.children is not actually used here */ + constructor(options: VerkleNodeOptions[VerkleNodeType.Internal]) { + super(options) + this.children = options.children ?? new Array(NODE_WIDTH).fill(null) + this.copyOnWrite = options.copyOnWrite ?? {} + } + + commit(): Point { + throw new Error('Not implemented') + } + + cowChild(_: number): void { + // Not implemented yet + } + + setChild(index: number, child: VerkleNode) { + this.children[index] = child + } + + static fromRawNode(rawNode: Uint8Array[], depth: number): InternalNode { + const nodeType = rawNode[0][0] + if (nodeType !== VerkleNodeType.Internal) { + throw new Error('Invalid node type') + } + + // The length of the rawNode should be the # of children, + 2 for the node type and the commitment + if (rawNode.length !== NODE_WIDTH + 2) { + throw new Error('Invalid node length') + } + + // TODO: Generate Point from rawNode value + const commitment = rawNode[rawNode.length - 1] as unknown as Point + + return new InternalNode({ commitment, depth }) + } + + static create(depth: number): InternalNode { + const node = new InternalNode({ + commitment: POINT_IDENTITY, + depth, + }) + + return node + } + + getChildren(index: number): VerkleNode | null { + return this.children?.[index] ?? null + } + + insert(key: Uint8Array, value: Uint8Array, resolver: () => void): void { + const values = new Array(NODE_WIDTH) + values[key[31]] = value + this.insertStem(key.slice(0, 31), values, resolver) + } + + insertStem(stem: Uint8Array, values: Uint8Array[], resolver: () => void): void { + // Index of the child pointed by the next byte in the key + const childIndex = stem[this.depth] + + const child = this.children[childIndex] + + if (child instanceof LeafNode) { + this.cowChild(childIndex) + if (equalsBytes(child.stem, stem)) { + return child.insertMultiple(stem, values) + } + + // A new branch node has to be inserted. Depending + // on the next byte in both keys, a recursion into + // the moved leaf node can occur. + const nextByteInExistingKey = child.stem[this.depth + 1] + const newBranch = InternalNode.create(this.depth + 1) + newBranch.cowChild(nextByteInExistingKey) + this.children[childIndex] = newBranch + newBranch.children[nextByteInExistingKey] = child + child.depth += 1 + + const nextByteInInsertedKey = stem[this.depth + 1] + if (nextByteInInsertedKey === nextByteInExistingKey) { + return newBranch.insertStem(stem, values, resolver) + } + + // Next word differs, so this was the last level. + // Insert it directly into its final slot. + const leafNode = LeafNode.create(stem, values) + + leafNode.setDepth(this.depth + 2) + newBranch.cowChild(nextByteInInsertedKey) + newBranch.children[nextByteInInsertedKey] = leafNode + } else if (child instanceof InternalNode) { + this.cowChild(childIndex) + return child.insertStem(stem, values, resolver) + } else { + throw new Error('Invalid node type') + } + } + + // TODO: go-verkle also adds the bitlist to the raw format. + raw(): Uint8Array[] { + throw new Error('not implemented yet') + // return [new Uint8Array([VerkleNodeType.Internal]), ...this.children, this.commitment] + } +} diff --git a/packages/verkle/src/node/leafNode.ts b/packages/verkle/src/node/leafNode.ts new file mode 100644 index 0000000000..dd11308d82 --- /dev/null +++ b/packages/verkle/src/node/leafNode.ts @@ -0,0 +1,85 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { BaseVerkleNode } from './baseVerkleNode.js' +import { NODE_WIDTH, VerkleNodeType } from './types.js' + +import type { Point } from '../types.js' +import type { VerkleNodeOptions } from './types.js' + +export class LeafNode extends BaseVerkleNode { + public stem: Uint8Array + public values: Uint8Array[] + public c1: Point + public c2: Point + public type = VerkleNodeType.Leaf + + constructor(options: VerkleNodeOptions[VerkleNodeType.Leaf]) { + super(options) + + this.stem = options.stem + this.values = options.values + this.c1 = options.c1 + this.c2 = options.c2 + } + + static create(stem: Uint8Array, values: Uint8Array[]): LeafNode { + throw new Error('Not implemented') + } + + static fromRawNode(rawNode: Uint8Array[], depth: number): LeafNode { + const nodeType = rawNode[0][0] + if (nodeType !== VerkleNodeType.Leaf) { + throw new Error('Invalid node type') + } + + // The length of the rawNode should be the # of values (node width) + 5 for the node type, the stem, the commitment and the 2 commitments + if (rawNode.length !== NODE_WIDTH + 5) { + throw new Error('Invalid node length') + } + + const stem = rawNode[1] + // TODO: Convert the rawNode commitments to points + const commitment = rawNode[2] as unknown as Point + const c1 = rawNode[3] as unknown as Point + const c2 = rawNode[4] as unknown as Point + const values = rawNode.slice(5, rawNode.length) + + return new LeafNode({ depth, stem, values, c1, c2, commitment }) + } + commit(): Point { + throw new Error('Not implemented') + } + + getValue(index: number): Uint8Array | null { + return this.values?.[index] ?? null + } + + insert(key: Uint8Array, value: Uint8Array, nodeResolverFn: () => void): void { + const values = new Array(NODE_WIDTH) + values[key[31]] = value + this.insertStem(key.slice(0, 31), values, nodeResolverFn) + } + + insertMultiple(key: Uint8Array, values: Uint8Array[]): void { + throw new Error('Not implemented') + } + + insertStem(key: Uint8Array, value: Uint8Array[], resolver: () => void): void { + throw new Error('Not implemented') + } + + // TODO: go-verkle also adds the bitlist to the raw format. + raw(): Uint8Array[] { + return [ + new Uint8Array([VerkleNodeType.Leaf]), + this.stem, + this.commitment.bytes(), + this.c1.bytes(), + this.c2.bytes(), + ...this.values, + ] + } + + setDepth(depth: number): void { + this.depth = depth + } +} diff --git a/packages/verkle/src/node/types.ts b/packages/verkle/src/node/types.ts new file mode 100644 index 0000000000..db975a5e5b --- /dev/null +++ b/packages/verkle/src/node/types.ts @@ -0,0 +1,49 @@ +import type { Point } from '../types.js' +import type { InternalNode } from './internalNode.js' +import type { LeafNode } from './leafNode.js' + +export enum VerkleNodeType { + Internal, + Leaf, +} + +export interface TypedVerkleNode { + [VerkleNodeType.Internal]: InternalNode + [VerkleNodeType.Leaf]: LeafNode +} + +export type VerkleNode = TypedVerkleNode[VerkleNodeType] + +export interface VerkleNodeInterface { + commit(): Point + hash(): any + serialize(): Uint8Array +} + +interface BaseVerkleNodeOptions { + // Value of the commitment + commitment: Point + depth: number +} + +interface VerkleInternalNodeOptions extends BaseVerkleNodeOptions { + // Children nodes of this internal node. + children?: VerkleNode[] + + // Values of the child commitments before the trie is modified by inserts. + // This is useful because the delta of the child commitments can be used to efficiently update the node's commitment + copyOnWrite?: Record +} +interface VerkleLeafNodeOptions extends BaseVerkleNodeOptions { + stem: Uint8Array + values: Uint8Array[] + c1: Point + c2: Point +} + +export interface VerkleNodeOptions { + [VerkleNodeType.Internal]: VerkleInternalNodeOptions + [VerkleNodeType.Leaf]: VerkleLeafNodeOptions +} + +export const NODE_WIDTH = 256 diff --git a/packages/verkle/src/node/util.ts b/packages/verkle/src/node/util.ts new file mode 100644 index 0000000000..16726076dc --- /dev/null +++ b/packages/verkle/src/node/util.ts @@ -0,0 +1,30 @@ +import { RLP } from '@ethereumjs/rlp' + +import { InternalNode } from './internalNode.js' +import { LeafNode } from './leafNode.js' +import { type VerkleNode, VerkleNodeType } from './types.js' + +export function decodeRawNode(raw: Uint8Array[]): VerkleNode { + const nodeType = raw[0][0] + const depth = 0 + switch (nodeType) { + case VerkleNodeType.Internal: + return InternalNode.fromRawNode(raw, depth) + case VerkleNodeType.Leaf: + return LeafNode.fromRawNode(raw, depth) + default: + throw new Error('Invalid node type') + } +} + +export function decodeNode(raw: Uint8Array) { + const decoded = RLP.decode(Uint8Array.from(raw)) as Uint8Array[] + if (!Array.isArray(decoded)) { + throw new Error('Invalid node') + } + return decodeRawNode(decoded) +} + +export function isRawNode(node: Uint8Array | Uint8Array[]): node is Uint8Array[] { + return Array.isArray(node) && !(node instanceof Uint8Array) +} diff --git a/packages/verkle/src/rust-verkle-wasm/LICENSE_APACHE b/packages/verkle/src/rust-verkle-wasm/LICENSE_APACHE new file mode 100644 index 0000000000..1b5ec8b78e --- /dev/null +++ b/packages/verkle/src/rust-verkle-wasm/LICENSE_APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/packages/verkle/src/rust-verkle-wasm/LICENSE_MIT b/packages/verkle/src/rust-verkle-wasm/LICENSE_MIT new file mode 100644 index 0000000000..f436a5ae91 --- /dev/null +++ b/packages/verkle/src/rust-verkle-wasm/LICENSE_MIT @@ -0,0 +1,25 @@ +Copyright (c) 2018 Kevaundray Wedderburn + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm.d.ts b/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm.d.ts new file mode 100644 index 0000000000..956249d273 --- /dev/null +++ b/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm.d.ts @@ -0,0 +1,18 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * @param {Uint8Array} address_tree_index + * @returns {any} + */ +export function pedersen_hash(address_tree_index: Uint8Array): any +/** + * @param {Uint8Array} js_root + * @param {Uint8Array} js_proof + * @param {Map} js_key_values + * @returns {any} + */ +export function verify_update( + js_root: Uint8Array, + js_proof: Uint8Array, + js_key_values: Map +): any diff --git a/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm.js b/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm.js new file mode 100644 index 0000000000..75d2edf53f --- /dev/null +++ b/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm.js @@ -0,0 +1,340 @@ +let imports = {} +imports['__wbindgen_placeholder__'] = module.exports +let wasm +const { TextEncoder, TextDecoder } = require(`util`) + +const heap = new Array(32).fill(undefined) + +heap.push(undefined, null, true, false) + +function getObject(idx) { + return heap[idx] +} + +let heap_next = heap.length + +function dropObject(idx) { + if (idx < 36) return + heap[idx] = heap_next + heap_next = idx +} + +function takeObject(idx) { + const ret = getObject(idx) + dropObject(idx) + return ret +} + +function debugString(val) { + // primitive types + const type = typeof val + if (type == 'number' || type == 'boolean' || val == null) { + return `${val}` + } + if (type == 'string') { + return `"${val}"` + } + if (type == 'symbol') { + const description = val.description + if (description == null) { + return 'Symbol' + } else { + return `Symbol(${description})` + } + } + if (type == 'function') { + const name = val.name + if (typeof name == 'string' && name.length > 0) { + return `Function(${name})` + } else { + return 'Function' + } + } + // objects + if (Array.isArray(val)) { + const length = val.length + let debug = '[' + if (length > 0) { + debug += debugString(val[0]) + } + for (let i = 1; i < length; i++) { + debug += ', ' + debugString(val[i]) + } + debug += ']' + return debug + } + // Test for built-in + const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val)) + let className + if (builtInMatches.length > 1) { + className = builtInMatches[1] + } else { + // Failed to match the standard '[object ClassName]' + return toString.call(val) + } + if (className == 'Object') { + // we're a user defined class or Object + // JSON.stringify avoids problems with cycles, and is generally much + // easier than looping through ownProperties of `val`. + try { + return 'Object(' + JSON.stringify(val) + ')' + } catch (_) { + return 'Object' + } + } + // errors + if (val instanceof Error) { + return `${val.name}: ${val.message}\n${val.stack}` + } + // TODO we could test for more things here, like `Set`s and `Map`s. + return className +} + +let WASM_VECTOR_LEN = 0 + +let cachedUint8Memory0 = new Uint8Array() + +function getUint8Memory0() { + if (cachedUint8Memory0.byteLength === 0) { + cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer) + } + return cachedUint8Memory0 +} + +let cachedTextEncoder = new TextEncoder('utf-8') + +const encodeString = + typeof cachedTextEncoder.encodeInto === 'function' + ? function (arg, view) { + return cachedTextEncoder.encodeInto(arg, view) + } + : function (arg, view) { + const buf = cachedTextEncoder.encode(arg) + view.set(buf) + return { + read: arg.length, + written: buf.length, + } + } + +function passStringToWasm0(arg, malloc, realloc) { + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg) + const ptr = malloc(buf.length) + getUint8Memory0() + .subarray(ptr, ptr + buf.length) + .set(buf) + WASM_VECTOR_LEN = buf.length + return ptr + } + + let len = arg.length + let ptr = malloc(len) + + const mem = getUint8Memory0() + + let offset = 0 + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset) + if (code > 0x7f) break + mem[ptr + offset] = code + } + + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset) + } + ptr = realloc(ptr, len, (len = offset + arg.length * 3)) + const view = getUint8Memory0().subarray(ptr + offset, ptr + len) + const ret = encodeString(arg, view) + + offset += ret.written + } + + WASM_VECTOR_LEN = offset + return ptr +} + +let cachedInt32Memory0 = new Int32Array() + +function getInt32Memory0() { + if (cachedInt32Memory0.byteLength === 0) { + cachedInt32Memory0 = new Int32Array(wasm.memory.buffer) + } + return cachedInt32Memory0 +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) + +cachedTextDecoder.decode() + +function getStringFromWasm0(ptr, len) { + return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len)) +} + +function addHeapObject(obj) { + if (heap_next === heap.length) heap.push(heap.length + 1) + const idx = heap_next + heap_next = heap[idx] + + heap[idx] = obj + return idx +} +/** + * @param {Uint8Array} address_tree_index + * @returns {any} + */ +module.exports.pedersen_hash = function (address_tree_index) { + const ret = wasm.pedersen_hash(addHeapObject(address_tree_index)) + return takeObject(ret) +} + +let stack_pointer = 32 + +function addBorrowedObject(obj) { + if (stack_pointer == 1) throw new Error('out of js stack') + heap[--stack_pointer] = obj + return stack_pointer +} +/** + * @param {Uint8Array} js_root + * @param {Uint8Array} js_proof + * @param {Map} js_key_values + * @returns {any} + */ +module.exports.verify_update = function (js_root, js_proof, js_key_values) { + try { + const ret = wasm.verify_update( + addHeapObject(js_root), + addHeapObject(js_proof), + addBorrowedObject(js_key_values) + ) + return takeObject(ret) + } finally { + heap[stack_pointer++] = undefined + } +} + +function handleError(f, args) { + try { + return f.apply(this, args) + } catch (e) { + wasm.__wbindgen_exn_store(addHeapObject(e)) + } +} + +module.exports.__wbindgen_object_drop_ref = function (arg0) { + takeObject(arg0) +} + +module.exports.__wbg_log_035529d7f1f4615f = function (arg0, arg1) { + console.log(getStringFromWasm0(arg0, arg1)) +} + +module.exports.__wbindgen_is_null = function (arg0) { + const ret = getObject(arg0) === null + return ret +} + +module.exports.__wbg_new_abda76e883ba8a5f = function () { + const ret = new Error() + return addHeapObject(ret) +} + +module.exports.__wbg_stack_658279fe44541cf6 = function (arg0, arg1) { + const ret = getObject(arg1).stack + const ptr0 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc) + const len0 = WASM_VECTOR_LEN + getInt32Memory0()[arg0 / 4 + 1] = len0 + getInt32Memory0()[arg0 / 4 + 0] = ptr0 +} + +module.exports.__wbg_error_f851667af71bcfc6 = function (arg0, arg1) { + try { + console.error(getStringFromWasm0(arg0, arg1)) + } finally { + wasm.__wbindgen_free(arg0, arg1) + } +} + +module.exports.__wbg_get_57245cc7d7c7619d = function (arg0, arg1) { + const ret = getObject(arg0)[arg1 >>> 0] + return addHeapObject(ret) +} + +module.exports.__wbg_next_aaef7c8aa5e212ac = function () { + return handleError(function (arg0) { + const ret = getObject(arg0).next() + return addHeapObject(ret) + }, arguments) +} + +module.exports.__wbg_done_1b73b0672e15f234 = function (arg0) { + const ret = getObject(arg0).done + return ret +} + +module.exports.__wbg_value_1ccc36bc03462d71 = function (arg0) { + const ret = getObject(arg0).value + return addHeapObject(ret) +} + +module.exports.__wbg_from_7ce3cb27cb258569 = function (arg0) { + const ret = Array.from(getObject(arg0)) + return addHeapObject(ret) +} + +module.exports.__wbg_entries_ff7071308de9aaec = function (arg0) { + const ret = getObject(arg0).entries() + return addHeapObject(ret) +} + +module.exports.__wbg_buffer_3f3d764d4747d564 = function (arg0) { + const ret = getObject(arg0).buffer + return addHeapObject(ret) +} + +module.exports.__wbg_newwithbyteoffsetandlength_d9aa266703cb98be = function (arg0, arg1, arg2) { + const ret = new Uint8Array(getObject(arg0), arg1 >>> 0, arg2 >>> 0) + return addHeapObject(ret) +} + +module.exports.__wbg_new_8c3f0052272a457a = function (arg0) { + const ret = new Uint8Array(getObject(arg0)) + return addHeapObject(ret) +} + +module.exports.__wbg_set_83db9690f9353e79 = function (arg0, arg1, arg2) { + getObject(arg0).set(getObject(arg1), arg2 >>> 0) +} + +module.exports.__wbg_length_9e1ae1900cb0fbd5 = function (arg0) { + const ret = getObject(arg0).length + return ret +} + +module.exports.__wbindgen_debug_string = function (arg0, arg1) { + const ret = debugString(getObject(arg1)) + const ptr0 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc) + const len0 = WASM_VECTOR_LEN + getInt32Memory0()[arg0 / 4 + 1] = len0 + getInt32Memory0()[arg0 / 4 + 0] = ptr0 +} + +module.exports.__wbindgen_throw = function (arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)) +} + +module.exports.__wbindgen_memory = function () { + const ret = wasm.memory + return addHeapObject(ret) +} + +const path = require('path').join(__dirname, 'rust_verkle_wasm_bg.wasm') +const bytes = require('fs').readFileSync(path) + +const wasmModule = new WebAssembly.Module(bytes) +const wasmInstance = new WebAssembly.Instance(wasmModule, imports) +wasm = wasmInstance.exports +module.exports.__wasm = wasm diff --git a/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm_bg.wasm b/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm_bg.wasm new file mode 100644 index 0000000000..a5d4fd9830 Binary files /dev/null and b/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm_bg.wasm differ diff --git a/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm_bg.wasm.d.ts b/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm_bg.wasm.d.ts new file mode 100644 index 0000000000..2276a26083 --- /dev/null +++ b/packages/verkle/src/rust-verkle-wasm/rust_verkle_wasm_bg.wasm.d.ts @@ -0,0 +1,9 @@ +/* tslint:disable */ +/* eslint-disable */ +export const memory: WebAssembly.Memory +export function pedersen_hash(a: number): number +export function verify_update(a: number, b: number, c: number): number +export function __wbindgen_malloc(a: number): number +export function __wbindgen_realloc(a: number, b: number, c: number): number +export function __wbindgen_free(a: number, b: number): void +export function __wbindgen_exn_store(a: number): void diff --git a/packages/verkle/src/types.ts b/packages/verkle/src/types.ts new file mode 100644 index 0000000000..3b946019b6 --- /dev/null +++ b/packages/verkle/src/types.ts @@ -0,0 +1,118 @@ +import { utf8ToBytes } from '@ethereumjs/util' + +import type { VerkleNode } from './node' +import type { WalkController } from './util/walkController' +import type { DB } from '@ethereumjs/util' + +// Field representation of a commitment +export interface Fr {} + +// Elliptic curve point representation of a commitment +export interface Point { + // Bytes returns the compressed serialized version of the element. + bytes(): Uint8Array + // BytesUncompressed returns the uncompressed serialized version of the element. + bytesUncompressed(): Uint8Array + + // SetBytes deserializes a compressed group element from buf. + // This method does all the proper checks assuming the bytes come from an + // untrusted source. + setBytes(bytes: Uint8Array): void + + // SetBytesUncompressed deserializes an uncompressed group element from buf. + setBytesUncompressed(bytes: Uint8Array, trusted: boolean): void + + // computes X/Y + mapToBaseField(): Point + + // mapToScalarField maps a group element to the scalar field. + mapToScalarField(field: Fr): void + + // Equal returns true if p and other represent the same point. + equal(secondPoint: Point): boolean + + // SetIdentity sets p to the identity element. + setIdentity(): Point + + // Double sets p to 2*p1. + double(point1: Point): Point + + // Add sets p to p1+p2. + add(point1: Point, point2: Point): Point + + // Sub sets p to p1-p2. + sub(point1: Point, point2: Point): Point + + // IsOnCurve returns true if p is on the curve. + isOnCurve(): boolean + + normalise(): void + + // Set sets p to p1. + set(): Point + + // Neg sets p to -p1. + neg(): Point + + // ScalarMul sets p to p1*s. + scalarMul(point1: Point, scalarMont: Fr): Point +} + +export type Proof = Uint8Array[] + +export interface VerkleTrieOpts { + /** + * A database instance. + */ + db?: DB + + /** + * A `Uint8Array` for the root of a previously stored trie + */ + root?: Uint8Array + + /** + * Store the root inside the database after every `write` operation + */ + useRootPersistence?: boolean + + /** + * LRU cache for trie nodes to allow for faster node retrieval. + * + * Default: 0 (deactivated) + */ + cacheSize?: number +} + +export type VerkleTrieOptsWithDefaults = VerkleTrieOpts & { + useRootPersistence: boolean + cacheSize: number +} + +export interface CheckpointDBOpts { + /** + * A database instance. + */ + db: DB + + /** + * Cache size (default: 0) + */ + cacheSize?: number +} + +export type Checkpoint = { + // We cannot use a Uint8Array => Uint8Array map directly. If you create two Uint8Arrays with the same internal value, + // then when setting a value on the Map, it actually creates two indices. + keyValueMap: Map + root: Uint8Array +} + +export type FoundNodeFunction = ( + nodeRef: Uint8Array, + node: VerkleNode | null, + key: Uint8Array, + walkController: WalkController +) => void + +export const ROOT_DB_KEY = utf8ToBytes('__root__') diff --git a/packages/verkle/src/util/bytes.ts b/packages/verkle/src/util/bytes.ts new file mode 100644 index 0000000000..9a5b83da93 --- /dev/null +++ b/packages/verkle/src/util/bytes.ts @@ -0,0 +1,25 @@ +/** + * Compares two byte arrays and returns the count of consecutively matching items from the start. + * + * @function + * @param {Uint8Array} bytes1 - The first Uint8Array to compare. + * @param {Uint8Array} bytes2 - The second Uint8Array to compare. + * @returns {number} The count of consecutively matching items from the start. + */ +export function matchingBytesLength(bytes1: Uint8Array, bytes2: Uint8Array): number { + let count = 0 + + // The minimum length of both arrays + const minLength = Math.min(bytes1.length, bytes2.length) + + for (let i = 0; i < minLength; i++) { + if (bytes1[i] === bytes2[i]) { + count++ + } else { + // Stop counting as soon as a mismatch is found + break + } + } + + return count +} diff --git a/packages/verkle/src/util/crypto.ts b/packages/verkle/src/util/crypto.ts new file mode 100644 index 0000000000..8143765b7b --- /dev/null +++ b/packages/verkle/src/util/crypto.ts @@ -0,0 +1,40 @@ +import { type Address, concatBytes, int32ToBytes, setLengthLeft, toBytes } from '@ethereumjs/util' + +import * as rustVerkleWasm from '../rust-verkle-wasm/rust_verkle_wasm.js' + +import type { Point } from '../types.js' + +export function pedersenHash(input: Uint8Array): Uint8Array { + const pedersenHash = rustVerkleWasm.pedersen_hash(input) + + if (pedersenHash === null) { + throw new Error( + 'pedersenHash: Wrong pedersenHash input. This might happen if length is not correct.' + ) + } + + return pedersenHash +} + +/** + * @dev Returns the tree key for a given address, tree index, and sub index. + * @dev Assumes that the verkle node width = 256 + * @param address The address to generate the tree key for. + * @param treeIndex The index of the tree to generate the key for. + * @param subIndex The sub index of the tree to generate the key for. + * @return The tree key as a Uint8Array. + */ +export function getTreeKey(address: Address, treeIndex: number, subIndex: number): Uint8Array { + const address32 = setLengthLeft(address.toBytes(), 32) + + const treeIndexB = int32ToBytes(treeIndex, true) + + const input = concatBytes(address32, treeIndexB) + + const treeKey = concatBytes(pedersenHash(input).slice(0, 31), toBytes(subIndex)) + + return treeKey +} + +// TODO: Replace this by the actual value of Point().Identity() from the Go code. +export const POINT_IDENTITY = new Uint8Array(32).fill(0) as unknown as Point diff --git a/packages/verkle/src/util/index.ts b/packages/verkle/src/util/index.ts new file mode 100644 index 0000000000..972e26b441 --- /dev/null +++ b/packages/verkle/src/util/index.ts @@ -0,0 +1,4 @@ +export * from './bytes.js' +export * from './crypto.js' +export * from './tasks.js' +export * from './walkController.js' diff --git a/packages/verkle/src/util/tasks.ts b/packages/verkle/src/util/tasks.ts new file mode 100644 index 0000000000..26dc56a96a --- /dev/null +++ b/packages/verkle/src/util/tasks.ts @@ -0,0 +1,59 @@ +interface Task { + priority: number + fn: Function +} + +export class PrioritizedTaskExecutor { + /** The maximum size of the pool */ + private maxPoolSize: number + /** The current size of the pool */ + private currentPoolSize: number + /** The task queue */ + private queue: Task[] + + /** + * Executes tasks up to maxPoolSize at a time, other items are put in a priority queue. + * @class PrioritizedTaskExecutor + * @private + * @param maxPoolSize The maximum size of the pool + */ + constructor(maxPoolSize: number) { + this.maxPoolSize = maxPoolSize + this.currentPoolSize = 0 + this.queue = [] + } + + /** + * Executes the task or queues it if no spots are available. + * When a task is added, check if there are spots left in the pool. + * If a spot is available, claim that spot and give back the spot once the asynchronous task has been resolved. + * When no spots are available, add the task to the task queue. The task will be executed at some point when another task has been resolved. + * @private + * @param priority The priority of the task + * @param fn The function that accepts the callback, which must be called upon the task completion. + */ + executeOrQueue(priority: number, fn: Function) { + if (this.currentPoolSize < this.maxPoolSize) { + this.currentPoolSize++ + fn(() => { + this.currentPoolSize-- + if (this.queue.length > 0) { + this.queue.sort((a, b) => b.priority - a.priority) + const item = this.queue.shift() + this.executeOrQueue(item!.priority, item!.fn) + } + }) + } else { + this.queue.push({ priority, fn }) + } + } + + /** + * Checks if the taskExecutor is finished. + * @private + * @returns Returns `true` if the taskExecutor is finished, otherwise returns `false`. + */ + finished(): boolean { + return this.currentPoolSize === 0 + } +} diff --git a/packages/verkle/src/util/walkController.ts b/packages/verkle/src/util/walkController.ts new file mode 100644 index 0000000000..1a8bbdbc4f --- /dev/null +++ b/packages/verkle/src/util/walkController.ts @@ -0,0 +1,145 @@ +import { InternalNode, LeafNode } from '../node/index.js' + +import { PrioritizedTaskExecutor } from './tasks.js' + +import type { VerkleNode } from '../node/types.js' +import type { FoundNodeFunction } from '../types.js' +import type { VerkleTrie } from '../verkleTrie.js' + +/** + * WalkController is an interface to control how the trie is being traversed. + */ +export class WalkController { + readonly onNode: FoundNodeFunction + readonly taskExecutor: PrioritizedTaskExecutor + readonly trie: VerkleTrie + private resolve: Function + private reject: Function + + /** + * Creates a new WalkController + * @param onNode - The `FoundNodeFunction` to call if a node is found. + * @param trie - The `VerkleTrie` to walk on. + * @param poolSize - The size of the task queue. + */ + private constructor(onNode: FoundNodeFunction, trie: VerkleTrie, poolSize: number) { + this.onNode = onNode + this.taskExecutor = new PrioritizedTaskExecutor(poolSize) + this.trie = trie + this.resolve = () => {} + this.reject = () => {} + } + + /** + * Async function to create and start a new walk over a trie. + * @param onNode - The `FoundNodeFunction to call if a node is found. + * @param trie - The trie to walk on. + * @param root - The root key to walk on. + * @param poolSize - Task execution pool size to prevent OOM errors. Defaults to 500. + */ + static async newWalk( + onNode: FoundNodeFunction, + trie: VerkleTrie, + root: Uint8Array, + poolSize?: number + ): Promise { + const strategy = new WalkController(onNode, trie, poolSize ?? 500) + await strategy.startWalk(root) + } + + private async startWalk(root: Uint8Array): Promise { + // eslint-disable-next-line no-async-promise-executor + return new Promise(async (resolve, reject) => { + this.resolve = resolve + this.reject = reject + let node + try { + node = await this.trie.lookupNode(root) + } catch (error) { + return this.reject(error) + } + this.processNode(root, node, new Uint8Array(0)) + }) + } + + /** + * Run all children of a node. Priority of these nodes are the key length of the children. + * @param node - Node to retrieve all children from of and call onNode on. + * @param key - The current `key` which would yield the `node` when trying to get this node with a `get` operation. + */ + allChildren(node: VerkleNode, key: Uint8Array = new Uint8Array()) { + if (node instanceof LeafNode) { + return + } + + const children = node.children.map((nodeRef, index) => ({ + keyExtension: index, + nodeRef, + })) + + for (const child of children) { + if (child.nodeRef !== null) { + const childKey = new Uint8Array([...key, child.keyExtension]) + this.pushNodeToQueue(child.nodeRef.hash(), childKey) + } + } + } + + /** + * Push a node to the queue. If the queue has places left for tasks, the node is executed immediately, otherwise it is queued. + * @param nodeRef - Push a node reference to the event queue. This reference is a 32-byte keccak hash of the value corresponding to the `key`. + * @param key - The current key. + * @param priority - Optional priority, defaults to key length + */ + pushNodeToQueue(nodeRef: Uint8Array, key: Uint8Array = new Uint8Array(0), priority?: number) { + this.taskExecutor.executeOrQueue( + priority ?? key.length, + async (taskFinishedCallback: Function) => { + let childNode + try { + childNode = await this.trie.lookupNode(nodeRef) + } catch (error: any) { + return this.reject(error) + } + taskFinishedCallback() // this marks the current task as finished. If there are any tasks left in the queue, this will immediately execute the first task. + this.processNode(nodeRef, childNode as VerkleNode, key) + } + ) + } + + /** + * Push the child of an internal node to the event queue. + * @param node - The node to select a children from. Should be an InternalNode. + * @param key - The current key which leads to the corresponding node. + * @param childIndex - The child index to add to the event queue. + * @param priority - Optional priority of the event, defaults to the total key length. + */ + pushChildrenAtIndex( + node: InternalNode, + key: Uint8Array = new Uint8Array(0), + childIndex: number, + priority?: number + ) { + if (!(node instanceof InternalNode)) { + throw new Error('Expected internal node') + } + const childRef = node.getChildren(childIndex) + if (!childRef) { + throw new Error('Could not get node at childIndex') + } + const childKey = new Uint8Array([...key, childIndex]) + this.pushNodeToQueue(childRef.hash(), childKey, priority ?? childKey.length) + } + + private processNode( + nodeRef: Uint8Array, + node: VerkleNode | null, + key: Uint8Array = new Uint8Array(0) + ) { + this.onNode(nodeRef, node, key, this) + if (this.taskExecutor.finished()) { + // onNode should schedule new tasks. If no tasks was added and the queue is empty, then we have finished our walk. + this.resolve() + } + } +} diff --git a/packages/verkle/src/verkleTrie.ts b/packages/verkle/src/verkleTrie.ts new file mode 100644 index 0000000000..51e90ac917 --- /dev/null +++ b/packages/verkle/src/verkleTrie.ts @@ -0,0 +1,516 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { + KeyEncoding, + Lock, + MapDB, + ValueEncoding, + bytesToHex, + equalsBytes, + hexToBytes, + zeros, +} from '@ethereumjs/util' + +import { CheckpointDB } from './db/checkpoint.js' +import { InternalNode } from './node/internalNode.js' +import { LeafNode } from './node/leafNode.js' +import { decodeNode, decodeRawNode, isRawNode } from './node/util.js' +import { + type Proof, + ROOT_DB_KEY, + type VerkleTrieOpts, + type VerkleTrieOptsWithDefaults, +} from './types.js' +import { WalkController, matchingBytesLength } from './util/index.js' + +import type { VerkleNode } from './node/types.js' +import type { FoundNodeFunction, Point } from './types.js' +import type { BatchDBOp, DB, PutBatch } from '@ethereumjs/util' + +interface Path { + node: VerkleNode | null + remaining: Uint8Array + stack: VerkleNode[] +} + +/** + * The basic verkle trie interface, use with `import { VerkleTrie } from '@ethereumjs/verkle'`. + */ +export class VerkleTrie { + protected readonly _opts: VerkleTrieOptsWithDefaults = { + useRootPersistence: false, + cacheSize: 0, + } + + /** The root for an empty trie */ + EMPTY_TRIE_ROOT: Uint8Array + + /** The backend DB */ + protected _db!: CheckpointDB + protected _hashLen: number + protected _lock = new Lock() + protected _root: Uint8Array + + /** + * Creates a new verkle trie. + * @param opts Options for instantiating the verkle trie + * + * Note: in most cases, the static {@link VerkleTrie.create} constructor should be used. It uses the same API but provides sensible defaults + */ + constructor(opts?: VerkleTrieOpts) { + if (opts !== undefined) { + this._opts = { ...this._opts, ...opts } + } + + this.database(opts?.db) + + this.EMPTY_TRIE_ROOT = zeros(32) + this._hashLen = this.EMPTY_TRIE_ROOT.length + this._root = this.EMPTY_TRIE_ROOT + + if (opts?.root) { + this.root(opts.root) + } + } + + static async create(opts?: VerkleTrieOpts) { + const key = ROOT_DB_KEY + + if (opts?.db !== undefined && opts?.useRootPersistence === true) { + if (opts?.root === undefined) { + opts.root = await opts?.db.get(key, { + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, + }) + } else { + await opts?.db.put(key, opts.root, { + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, + }) + } + } + + return new VerkleTrie(opts) + } + + database(db?: DB) { + if (db !== undefined) { + if (db instanceof CheckpointDB) { + throw new Error('Cannot pass in an instance of CheckpointDB') + } + + this._db = new CheckpointDB({ db, cacheSize: this._opts.cacheSize }) + } + + return this._db + } + + /** + * Gets and/or Sets the current root of the `trie` + */ + root(value?: Uint8Array | null): Uint8Array { + if (value !== undefined) { + if (value === null) { + value = this.EMPTY_TRIE_ROOT + } + + if (value.length !== this._hashLen) { + throw new Error(`Invalid root length. Roots are ${this._hashLen} bytes`) + } + + this._root = value + } + + return this._root + } + + /** + * Checks if a given root exists. + */ + async checkRoot(root: Uint8Array): Promise { + try { + const value = await this.lookupNode(root) + return value !== null + } catch (error: any) { + if (error.message === 'Missing node in DB') { + return equalsBytes(root, this.EMPTY_TRIE_ROOT) + } else { + throw error + } + } + } + + /** + * Gets a value given a `key` + * @param key - the key to search for + * @param throwIfMissing - if true, throws if any nodes are missing. Used for verifying proofs. (default: false) + * @returns A Promise that resolves to `Uint8Array` if a value was found or `null` if no value was found. + */ + async get(key: Uint8Array, throwIfMissing = false): Promise { + const node = await this.findLeafNode(key, throwIfMissing) + if (node !== null) { + const keyLastByte = key[key.length - 1] + + // The retrieved leaf node contains an array of 256 possible values. + // The index of the value we want is at the key's last byte + return node.values?.[keyLastByte] ?? null + } + + return null + } + + /** + * Stores a given `value` at the given `key` or do a delete if `value` is empty + * (delete operations are only executed on DB with `deleteFromDB` set to `true`) + * @param key - the key to store the value at + * @param value - the value to store + * @returns A Promise that resolves once value is stored. + */ + async put(key: Uint8Array, value: Uint8Array): Promise { + await this._db.put(key, value) + + // Find or create the leaf node + const leafNode = await this.findLeafNode(key, false) + if (leafNode === null) { + // If leafNode is missing, create it + // leafNode = LeafNode.create() + throw new Error('Not implemented') + } + + // Walk up the trie and update internal nodes + let currentNode: VerkleNode = leafNode + let currentKey = leafNode.stem + let currentDepth = leafNode.depth + + while (currentDepth > 0) { + const parentKey = currentKey.slice(0, -1) + const parentIndex = currentKey[currentKey.length - 1] + const parentNode = InternalNode.create(currentDepth) + parentNode.children[parentIndex] = currentNode + await this._db.put(parentKey, parentNode.serialize()) + + currentNode = parentNode + currentKey = parentKey + currentDepth-- + } + + this._root = currentNode.hash() + } + + /** + * Tries to find a path to the node for the given key. + * It returns a `stack` of nodes to the closest node. + * @param key - the search key + * @param throwIfMissing - if true, throws if any nodes are missing. Used for verifying proofs. (default: false) + */ + async findPath(key: Uint8Array, throwIfMissing = false): Promise { + // eslint-disable-next-line no-async-promise-executor + return new Promise(async (resolve, reject) => { + const stack: VerkleNode[] = [] + + const onFound: FoundNodeFunction = async (_, node, keyProgress, walkController) => { + if (node === null) { + return reject(new Error('Path not found')) + } + const keyRemainder = key.slice(matchingBytesLength(keyProgress, key)) + stack.push(node) + + if (node instanceof InternalNode) { + if (keyRemainder.length === 0) { + // we exhausted the key without finding a node + resolve({ node, remaining: new Uint8Array(0), stack }) + } else { + const childrenIndex = keyRemainder[0] + const childNode = node.getChildren(childrenIndex) + if (childNode === null) { + // There are no more nodes to find and we didn't find the key + resolve({ node: null, remaining: keyRemainder, stack }) + } else { + // node found, continue search from children + walkController.pushChildrenAtIndex(node, keyProgress, childrenIndex) + } + } + } else if (node instanceof LeafNode) { + // The stem of the leaf node should be the full key minus the last byte + const stem = key.slice(0, key.length - 1) + if (equalsBytes(stem, node.stem)) { + // keys match, return node with empty key + resolve({ node, remaining: new Uint8Array(0), stack }) + } else { + // reached leaf but keys don't match + resolve({ node: null, remaining: keyRemainder, stack }) + } + } + } + + // walk trie and process nodes + try { + await this.walkTrie(this.root(), onFound) + } catch (error: any) { + if (error.message === 'Missing node in DB' && !throwIfMissing) { + // pass + } else { + reject(error) + } + } + + // Resolve if walkTrie finishes without finding any nodes + resolve({ node: null, remaining: new Uint8Array(0), stack }) + }) + } + + /** + * Walks a trie until finished. + * @param root + * @param onFound - callback to call when a node is found. This schedules new tasks. If no tasks are available, the Promise resolves. + * @returns Resolves when finished walking trie. + */ + async walkTrie(root: Uint8Array, onFound: FoundNodeFunction): Promise { + await WalkController.newWalk(onFound, this, root) + } + + /** + * Tries to find the leaf node leading up to the given key, or null if not found. + * @param key - the search key + * @param throwIfMissing - if true, throws if any nodes are missing. Used for verifying proofs. (default: false) + */ + async findLeafNode(key: Uint8Array, throwIfMissing = false): Promise { + const { node } = await this.findPath(key, throwIfMissing) + if (!(node instanceof LeafNode)) { + if (throwIfMissing) { + throw new Error('leaf node not found') + } + return null + } + return node + } + + /** + * Creates the initial node from an empty tree. + * @private + */ + protected async _createInitialNode(key: Uint8Array, value: Uint8Array): Promise { + throw new Error('Not implemented') + } + + /** + * Retrieves a node from db by hash. + */ + async lookupNode(node: Uint8Array | Uint8Array[]): Promise { + if (isRawNode(node)) { + return decodeRawNode(node) + } + const value = await this._db.get(node) + if (value !== undefined) { + return decodeNode(value) + } else { + // Dev note: this error message text is used for error checking in `checkRoot`, `verifyProof`, and `findPath` + throw new Error('Missing node in DB') + } + } + + /** + * Updates a node. + * @private + * @param key + * @param value + * @param keyRemainder + * @param stack + */ + protected async _updateNode( + k: Uint8Array, + value: Uint8Array, + keyRemainder: Uint8Array, + stack: VerkleNode[] + ): Promise { + throw new Error('Not implemented') + } + + /** + * Saves a stack of nodes to the database. + * + * @param key - the key. Should follow the stack + * @param stack - a stack of nodes to the value given by the key + * @param opStack - a stack of levelup operations to commit at the end of this function + */ + async saveStack( + key: Uint8Array, + stack: VerkleNode[], + opStack: PutBatch[] + ): Promise { + throw new Error('Not implemented') + } + + /** + * Formats node to be saved by `levelup.batch`. + * @private + * @param node - the node to format. + * @param topLevel - if the node is at the top level. + * @param opStack - the opStack to push the node's data. + * @param remove - whether to remove the node + * @returns The node's hash used as the key or the rawNode. + */ + _formatNode( + node: VerkleNode, + topLevel: boolean, + opStack: PutBatch, + remove: boolean = false + ): Uint8Array { + throw new Error('Not implemented') + } + + /** + * The given hash of operations (key additions or deletions) are executed on the trie + * (delete operations are only executed on DB with `deleteFromDB` set to `true`) + * @example + * const ops = [ + * { type: 'del', key: Uint8Array.from('father') } + * , { type: 'put', key: Uint8Array.from('name'), value: Uint8Array.from('Yuri Irsenovich Kim') } + * , { type: 'put', key: Uint8Array.from('dob'), value: Uint8Array.from('16 February 1941') } + * , { type: 'put', key: Uint8Array.from('spouse'), value: Uint8Array.from('Kim Young-sook') } + * , { type: 'put', key: Uint8Array.from('occupation'), value: Uint8Array.from('Clown') } + * ] + * await trie.batch(ops) + * @param ops + */ + async batch(ops: BatchDBOp[]): Promise { + throw new Error('Not implemented') + } + + /** + * Saves the nodes from a proof into the trie. + * @param proof + */ + async fromProof(proof: Proof): Promise { + throw new Error('Not implemented') + } + + /** + * Creates a proof from a trie and key that can be verified using {@link Trie.verifyProof}. + * @param key + */ + async createProof(key: Uint8Array): Promise { + throw new Error('Not implemented') + } + + /** + * Verifies a proof. + * @param rootHash + * @param key + * @param proof + * @throws If proof is found to be invalid. + * @returns The value from the key, or null if valid proof of non-existence. + */ + async verifyProof( + rootHash: Uint8Array, + key: Uint8Array, + proof: Proof + ): Promise { + throw new Error('Not implemented') + } + + /** + * The `data` event is given an `Object` that has two properties; the `key` and the `value`. Both should be Uint8Arrays. + * @return Returns a [stream](https://nodejs.org/dist/latest-v12.x/docs/api/stream.html#stream_class_stream_readable) of the contents of the `trie` + */ + createReadStream(): any { + throw new Error('Not implemented') + } + + /** + * Returns a copy of the underlying trie. + * + * Note on db: the copy will create a reference to the + * same underlying database. + * + * Note on cache: for memory reasons a copy will not + * recreate a new LRU cache but initialize with cache + * being deactivated. + * + * @param includeCheckpoints - If true and during a checkpoint, the copy will contain the checkpointing metadata and will use the same scratch as underlying db. + */ + shallowCopy(includeCheckpoints = true): VerkleTrie { + const trie = new VerkleTrie({ + ...this._opts, + db: this._db.db.shallowCopy(), + root: this.root(), + cacheSize: 0, + }) + if (includeCheckpoints && this.hasCheckpoints()) { + trie._db.setCheckpoints(this._db.checkpoints) + } + return trie + } + + /** + * Persists the root hash in the underlying database + */ + async persistRoot() { + if (this._opts.useRootPersistence) { + await this._db.put(ROOT_DB_KEY, this.root()) + } + } + + /** + * Finds all nodes that are stored directly in the db + * (some nodes are stored raw inside other nodes) + * called by {@link ScratchReadStream} + * @private + */ + protected async _findDbNodes(onFound: () => void): Promise { + throw new Error('Not implemented') + } + + /** + * Is the trie during a checkpoint phase? + */ + hasCheckpoints() { + return this._db.hasCheckpoints() + } + + /** + * Creates a checkpoint that can later be reverted to or committed. + * After this is called, all changes can be reverted until `commit` is called. + */ + checkpoint() { + this._db.checkpoint(this.root()) + } + + /** + * Commits a checkpoint to disk, if current checkpoint is not nested. + * If nested, only sets the parent checkpoint as current checkpoint. + * @throws If not during a checkpoint phase + */ + async commit(): Promise { + if (!this.hasCheckpoints()) { + throw new Error('trying to commit when not checkpointed') + } + + await this._lock.acquire() + await this._db.commit() + await this.persistRoot() + this._lock.release() + } + + /** + * Reverts the trie to the state it was at when `checkpoint` was first called. + * If during a nested checkpoint, sets root to most recent checkpoint, and sets + * parent checkpoint as current. + */ + async revert(): Promise { + if (!this.hasCheckpoints()) { + throw new Error('trying to revert when not checkpointed') + } + + await this._lock.acquire() + this.root(await this._db.revert()) + await this.persistRoot() + this._lock.release() + } + + /** + * Flushes all checkpoints, restoring the initial checkpoint state. + */ + flushCheckpoints() { + this._db.checkpoints = [] + } +} diff --git a/packages/verkle/test/node/internalNode.spec.ts b/packages/verkle/test/node/internalNode.spec.ts new file mode 100644 index 0000000000..f27b375a86 --- /dev/null +++ b/packages/verkle/test/node/internalNode.spec.ts @@ -0,0 +1,52 @@ +import { equalsBytes, randomBytes } from '@ethereumjs/util' +import { assert, describe, it } from 'vitest' + +import { NODE_WIDTH, VerkleNodeType } from '../../src/node/index.js' +import { InternalNode } from '../../src/node/internalNode.js' +import { POINT_IDENTITY } from '../../src/util/crypto.js' + +import type { Point } from '../../src/types.js' + +describe('verkle node - internal', () => { + it('constructor should create an internal node', async () => { + const commitment = randomBytes(32) + const depth = 2 + const node = new InternalNode({ commitment: commitment as unknown as Point, depth }) + + assert.equal(node.type, VerkleNodeType.Internal, 'type should be set') + assert.ok( + equalsBytes(node.commitment as unknown as Uint8Array, commitment), + 'commitment should be set' + ) + assert.equal(node.depth, depth, 'depth should be set') + + // Children nodes should all default to null. + assert.equal(node.children.length, NODE_WIDTH, 'number of children should equal verkle width') + assert.ok( + node.children.every((child) => child === null), + 'every children should be null' + ) + }) + + it('create method should create an internal node', async () => { + const depth = 3 + const node = InternalNode.create(depth) + + assert.equal(node.type, VerkleNodeType.Internal, 'type should be set') + assert.ok( + equalsBytes( + node.commitment as unknown as Uint8Array, + POINT_IDENTITY as unknown as Uint8Array + ), + 'commitment should be set to point identity' + ) + assert.equal(node.depth, depth, 'depth should be set') + + // Children nodes should all default to null. + assert.equal(node.children.length, NODE_WIDTH, 'number of children should equal verkle width') + assert.ok( + node.children.every((child) => child === null), + 'every children should be null' + ) + }) +}) diff --git a/packages/verkle/test/node/leafNode.spec.ts b/packages/verkle/test/node/leafNode.spec.ts new file mode 100644 index 0000000000..251c4cc6c3 --- /dev/null +++ b/packages/verkle/test/node/leafNode.spec.ts @@ -0,0 +1,42 @@ +import { equalsBytes, randomBytes } from '@ethereumjs/util' +import { assert, describe, it } from 'vitest' + +import { VerkleNodeType } from '../../src/node/index.js' +import { LeafNode } from '../../src/node/leafNode.js' + +import type { Point } from '../../src/types.js' + +describe('verkle node - leaf', () => { + it('constructor should create an leaf node', async () => { + const commitment = randomBytes(32) + const c1 = randomBytes(32) + const c2 = randomBytes(32) + const stem = randomBytes(32) + const values = [randomBytes(32), randomBytes(32)] + const depth = 2 + const node = new LeafNode({ + c1: c1 as unknown as Point, + c2: c2 as unknown as Point, + commitment: commitment as unknown as Point, + depth, + stem, + values, + }) + + assert.equal(node.type, VerkleNodeType.Leaf, 'type should be set') + assert.ok( + equalsBytes(node.commitment as unknown as Uint8Array, commitment), + 'commitment should be set' + ) + assert.ok(equalsBytes(node.c1 as unknown as Uint8Array, c1), 'c1 should be set') + assert.ok(equalsBytes(node.c2 as unknown as Uint8Array, c2), 'c2 should be set') + assert.ok(equalsBytes(node.stem, stem), 'stem should be set') + assert.ok( + values.every((value, index) => equalsBytes(value, node.values[index])), + 'values should be set' + ) + assert.equal(node.depth, depth, 'depth should be set') + }) + + it.todo('create method should create an leaf node') +}) diff --git a/packages/verkle/test/verkle.spec.ts b/packages/verkle/test/verkle.spec.ts new file mode 100644 index 0000000000..6bccd73918 --- /dev/null +++ b/packages/verkle/test/verkle.spec.ts @@ -0,0 +1,60 @@ +import { equalsBytes, hexToBytes } from '@ethereumjs/util' +import { assert, describe, it } from 'vitest' + +import { VerkleTrie } from '../src/verkleTrie.js' + +// Testdata from https://github.com/gballet/go-ethereum/blob/kaustinen-with-shapella/trie/verkle_test.go +const presentKeys = [ + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d01', + '0xe6ed6c222e3985050b4fc574b136b0a42c63538e9ab970995cd418ba8e526400', + '0x18fb432d3b859ec3a1803854e8cceea75d092e52d0d4a4398d13022496745a02', + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d02', + '0x18fb432d3b859ec3a1803854e8cceea75d092e52d0d4a4398d13022496745a04', + '0xe6ed6c222e3985050b4fc574b136b0a42c63538e9ab970995cd418ba8e526402', + '0xe6ed6c222e3985050b4fc574b136b0a42c63538e9ab970995cd418ba8e526403', + '0x18fb432d3b859ec3a1803854e8cceea75d092e52d0d4a4398d13022496745a00', + '0x18fb432d3b859ec3a1803854e8cceea75d092e52d0d4a4398d13022496745a03', + '0xe6ed6c222e3985050b4fc574b136b0a42c63538e9ab970995cd418ba8e526401', + '0xe6ed6c222e3985050b4fc574b136b0a42c63538e9ab970995cd418ba8e526404', + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d00', + '0x18fb432d3b859ec3a1803854e8cceea75d092e52d0d4a4398d13022496745a01', +].map(hexToBytes) + +// Corresponding values for the present keys +const values = [ + '0x320122e8584be00d000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0300000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470', + '0x1bc176f2790c91e6000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0x0000000000000000000000000000000000000000000000000000000000000000', + '0xe703000000000000000000000000000000000000000000000000000000000000', +].map(hexToBytes) + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const absentKeys = [ + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d03', + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d04', +].map(hexToBytes) + +describe('Verkle trie', () => { + it.todo('should insert and retrieve values', async () => { + const trie = new VerkleTrie() + for (let i = 0; i < presentKeys.length; i++) { + await trie.put(presentKeys[i], values[i]) + } + for (let i = 0; i < presentKeys.length; i++) { + const retrievedValue = await trie.get(presentKeys[i]) + if (retrievedValue === null) { + assert.fail('Value not found') + } + assert.ok(equalsBytes(retrievedValue, values[i])) + } + }) +}) diff --git a/packages/verkle/tsconfig.benchmarks.json b/packages/verkle/tsconfig.benchmarks.json new file mode 100644 index 0000000000..87f297c0ec --- /dev/null +++ b/packages/verkle/tsconfig.benchmarks.json @@ -0,0 +1,4 @@ +{ + "extends": "../../config/tsconfig.prod.cjs.json", + "include": ["benchmarks/*.ts"] +} diff --git a/packages/verkle/tsconfig.json b/packages/verkle/tsconfig.json new file mode 100644 index 0000000000..03ee66c13b --- /dev/null +++ b/packages/verkle/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../config/tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + }, + "include": ["src/**/*.ts", "test/**/*.spec.ts"] +} diff --git a/packages/verkle/tsconfig.prod.cjs.json b/packages/verkle/tsconfig.prod.cjs.json new file mode 100644 index 0000000000..fb8608068a --- /dev/null +++ b/packages/verkle/tsconfig.prod.cjs.json @@ -0,0 +1,13 @@ +{ + "extends": "../../config/tsconfig.prod.cjs.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist/cjs", + "composite": true + }, + "include": ["src/**/*.ts"], + "references": [ + { "path": "../rlp/tsconfig.prod.cjs.json" }, + { "path": "../util/tsconfig.prod.cjs.json" } + ] +} diff --git a/packages/verkle/tsconfig.prod.esm.json b/packages/verkle/tsconfig.prod.esm.json new file mode 100644 index 0000000000..e24adc754d --- /dev/null +++ b/packages/verkle/tsconfig.prod.esm.json @@ -0,0 +1,13 @@ +{ + "extends": "../../config/tsconfig.prod.esm.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist/esm", + "composite": true + }, + "include": ["src/**/*.ts"], + "references": [ + { "path": "../rlp/tsconfig.prod.esm.json" }, + { "path": "../util/tsconfig.prod.esm.json" } + ] +} diff --git a/packages/verkle/typedoc.cjs b/packages/verkle/typedoc.cjs new file mode 100644 index 0000000000..701fee055f --- /dev/null +++ b/packages/verkle/typedoc.cjs @@ -0,0 +1,6 @@ +module.exports = { + extends: '../../config/typedoc.cjs', + entryPoints: ['src'], + out: 'docs', + exclude: ['test/**/*.ts'], +} diff --git a/packages/verkle/vitest.config.browser.ts b/packages/verkle/vitest.config.browser.ts new file mode 100644 index 0000000000..9803b3ac44 --- /dev/null +++ b/packages/verkle/vitest.config.browser.ts @@ -0,0 +1,7 @@ +import { configDefaults, defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + exclude: [...configDefaults.exclude], + }, +})