diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd8150a..8944c90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: branches: [ "main" ] env: - VERSION: 0.2.1 + VERSION: 0.3.0 jobs: node: diff --git a/cli.mjs b/cli.mjs index 3c00e60..fa4d65b 100644 --- a/cli.mjs +++ b/cli.mjs @@ -1,7 +1,9 @@ #!/usr/bin/env node import index from './index.mjs' -const { get, asyncFileProvider, fetchProvider } = index +import ioNode from './io/node.mjs' +const { getLocal, getRemote } = index +const { node } = ioNode var args = process.argv.slice(2) @@ -12,5 +14,5 @@ if (args.length < 2) { } const hostName = args[2] -const provider = hostName === undefined ? asyncFileProvider : fetchProvider(hostName) -get(provider)([args[0], args[1]]) \ No newline at end of file +const getFunc = hostName === undefined ? getLocal : getRemote(hostName) +getFunc(node)([args[0], args[1]]) \ No newline at end of file diff --git a/get.mjs b/get.mjs index b8500a2..868aab7 100644 --- a/get.mjs +++ b/get.mjs @@ -62,12 +62,6 @@ const { tailToDigest } = digest256 * }} Provider */ -/** @type {(address: Address) => string} */ -const getPath = ([address, isRoot]) => { - const dir = isRoot ? 'roots' : 'parts' - return `cdt0/${dir}/${address.substring(0, 2)}/${address.substring(2, 4)}/${address.substring(4)}` -} - /** @type {(state: State) => (block: Block) => void} */ const insertBlock = state => block => { for (let i = 0; i < state.length; i++) { @@ -138,10 +132,6 @@ const nextState = state => { } } -/** @type {(hostName: string) => (address: Address) => Promise} */ -const fetchRead = hostName => address => fetch(`https://${hostName}/${getPath(address)}`) - .then(async (resp) => resp.arrayBuffer().then(buffer => new Uint8Array(buffer))) - /** @type {(provider: Provider) => (root: string) => Promise} */ const get = ({ read, write }) => async (root) => { /** @type {State} */ @@ -194,6 +184,5 @@ const get = ({ read, write }) => async (root) => { } export default { - get, - fetchRead + get } \ No newline at end of file diff --git a/index.mjs b/index.mjs index e6be857..59be171 100644 --- a/index.mjs +++ b/index.mjs @@ -1,70 +1,13 @@ -import fs from 'node:fs' -import fsPromises from 'node:fs/promises' -import base32 from './base32.mjs' -import tree from './tree.mjs' -import digest256 from './digest256.mjs' import getModule from './get.mjs' /** @typedef {import('./tree.mjs').State} StateTree */ /** * @template T * @typedef {import('./subtree.mjs').Nullable} Nullable */ -const { toAddress } = base32 -const { push: pushTree, end: endTree, partialEnd: partialEndTree, pushDigest } = tree -const { tailToDigest } = digest256 -const { get, fetchRead } = getModule +/** @typedef {import('./get.mjs').Address} Address */ +/** @typedef {import('./io/io.mjs').IO} IO */ -/** - * second element is root flag - * @typedef {readonly [string, boolean]} Address - */ - -/** - * @typedef {[Address, Uint8Array]} Block - */ - -/** - * @template T - * @typedef {readonly['ok', T]} Ok - */ - -/** - * @template E - * @typedef {readonly['error', E]} Error - */ - -/** - * @template T - * @template E - * @typedef {Ok|Error} Result - */ - -/** - * @typedef {readonly Uint8Array[]} OkOutput - */ - -/** - * @typedef { Result } Output -*/ - -/** - * @typedef { Uint8Array } ReadonlyUint8Array - */ - -/** - * @typedef {[Address, Nullable]} BlockState - */ - -/** - * @typedef { BlockState[] } State -*/ - -/** - * @typedef {{ - * readonly read: (address: Address) => Promise, - * readonly write: (path: string) => (buffer: Uint8Array) => Promise, - * }} Provider -*/ +const { get } = getModule /** @type {(address: Address) => string} */ const getPath = ([address, isRoot]) => { @@ -72,112 +15,46 @@ const getPath = ([address, isRoot]) => { return `cdt0/${dir}/${address.substring(0, 2)}/${address.substring(2, 4)}/${address.substring(4)}` } -/** @type {(state: State) => (block: Block) => void} */ -const insertBlock = state => block => { - for (let i = 0; i < state.length; i++) { - if (state[i][0][0] === block[0][0]) { - state[i][1] = block[1] - } - } -} - -/** @type {(state: State) => Output} */ -const nextState = state => { - /** @type {Uint8Array[]} */ - let resultBuffer = [] - - while (true) { - const blockLast = state.at(-1) - if (blockLast === undefined) { - return ['ok', resultBuffer] - } - - const blockData = blockLast[1] - if (blockData === null) { - return ['ok', resultBuffer] - } - - state.pop() - - if (blockLast[0][0] === '') { - resultBuffer.push(blockData) - continue - } - - /** @type {StateTree} */ - let verificationTree = [] - const tailLength = blockData[0] - if (tailLength === 32) { - const data = blockData.subarray(1) - for (let byte of data) { - pushTree(verificationTree)(byte) - } - resultBuffer.push(data) - } else { - const tail = blockData.subarray(1, tailLength + 1) - if (tail.length !== 0) { - state.push([['', false], tail]) - } - /** @type {Address[]} */ - let childAddresses = [] - for (let i = tailLength + 1; i < blockData.length; i += 28) { - let hash = 0n - for (let j = 0; j < 28; j++) { - hash += BigInt(blockData[i + j]) << BigInt(8 * j) - } - pushDigest(verificationTree)(hash | (0xffff_ffffn << 224n)) - const childAddress = toAddress(hash) - childAddresses.push([childAddress, false]) - } - pushDigest(verificationTree)(tailToDigest(tail)) - const digest = blockLast[0][1] ? endTree(verificationTree) : partialEndTree(verificationTree) - if (digest === null || toAddress(digest) !== blockLast[0][0]) { - return ['error', `verification failed ${blockLast[0][0]}`] - } +/** @type {(hostName: string) => (io: IO) => (address: Address) => Promise} */ +const fetchRead = hostName => ({ fetch }) => address => fetch(`https://${hostName}/${getPath(address)}`) + .then(async (resp) => resp.arrayBuffer().then(buffer => new Uint8Array(buffer))) - for (let i = childAddresses.length - 1; i >= 0; i--) { - state.push([childAddresses[i], null]) - } - } +/** @type {(io: IO) => (root: [string, string]) => Promise} */ +const getLocal = io => async ([root, file]) => { + const tempFile = `_temp_${root}` + await io.write(tempFile, new Uint8Array()) + /** @type {(address: Address) => Promise} */ + const read = address => io.read(getPath(address)) + /** @type {(buffer: Uint8Array) => Promise} */ + const write = buffer => io.append(tempFile, buffer) + const error = await get({ read, write })(root) + if (error !== null) { + console.error(error) + return -1 } + await io.rename(tempFile, file) + return 0 } -/** @type {(hostName: string) => Provider} */ -const fetchProvider = hostName => ({ - read: fetchRead(hostName), - write: path => buffer => fsPromises.appendFile(path, buffer) -}) - -/** @type {Provider} */ -const asyncFileProvider = { - read: address => fsPromises.readFile(getPath(address)), - write: path => buffer => fsPromises.appendFile(path, buffer) -} - -/** @type {Provider} */ -const syncFileProvider = { - read: address => Promise.resolve(fs.readFileSync(getPath(address))), - write: path => buffer => Promise.resolve(fs.appendFileSync(path, buffer)) -} - -/** @type {(provider: Provider) => (root: [string, string]) => Promise} */ -const getLocal = ({ read, write }) => async ([root, file]) => { +/** @type {(host: string) => (io: IO) => (root: [string, string]) => Promise} */ +const getRemote = host => io => async ([root, file]) => { const tempFile = `_temp_${root}` - await fsPromises.writeFile(tempFile, new Uint8Array()) + await io.write(tempFile, new Uint8Array()) + /** @type {(address: Address) => Promise} */ + const read = fetchRead(host)(io) /** @type {(buffer: Uint8Array) => Promise} */ - const write = buffer => fsPromises.appendFile(tempFile, buffer) + const write = buffer => io.append(tempFile, buffer) const error = await get({ read, write })(root) if (error !== null) { console.error(error) return -1 } - await fsPromises.rename(tempFile, file) + await io.rename(tempFile, file) return 0 } export default { - get: getLocal, - syncFileProvider, - asyncFileProvider, - fetchProvider + getLocal, + getRemote, + fetchRead } \ No newline at end of file diff --git a/io/io.mjs b/io/io.mjs new file mode 100644 index 0000000..6efdc5c --- /dev/null +++ b/io/io.mjs @@ -0,0 +1,13 @@ +/** + * @typedef {{ +* readonly read: (path: string) => Promise, +* readonly append: (path: string, buffer: Uint8Array) => Promise, +* readonly write: (path: string, buffer: Uint8Array) => Promise, +* readonly rename: (oldPath: string, newPath: string) => Promise +* readonly fetch: (url: string) => Promise +* readonly document: Document | undefined +* }} IO +*/ + +export default { +} \ No newline at end of file diff --git a/io/node.mjs b/io/node.mjs new file mode 100644 index 0000000..6c6ef89 --- /dev/null +++ b/io/node.mjs @@ -0,0 +1,28 @@ +import fs from 'node:fs' +import fsPromises from 'node:fs/promises' +/** @typedef {import('./io.mjs').IO} IO */ + +/** @type {IO} */ +const node = { + read: fsPromises.readFile, + append: fsPromises.appendFile, + write: fsPromises.writeFile, + rename: fsPromises.rename, + fetch, + document: undefined +} + +/** @type {IO} */ +const nodeSync = { + read: async(path) => fs.readFileSync(path), + append: async(path, buffer) => fs.appendFileSync(path, buffer), + write: async(path, buffer) => fs.writeFileSync(path, buffer), + rename: async(oldPath, newPath) => fs.renameSync(oldPath, newPath), + fetch, + document: undefined +} + +export default { + node, + nodeSync +} \ No newline at end of file diff --git a/io/web.mjs b/io/web.mjs new file mode 100644 index 0000000..af4512e --- /dev/null +++ b/io/web.mjs @@ -0,0 +1,17 @@ +/** @typedef {import('./io.mjs').IO} IO */ + +const notImplemented = () => { throw 'not implemented' } + +/** @type {IO} */ +const web = { + read: notImplemented, + append: notImplemented, + write: notImplemented, + rename: notImplemented, + fetch, + document +} + +export default { + web +} \ No newline at end of file diff --git a/package.json b/package.json index 4125cdc..299f782 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "blockset-js", - "version": "0.2.1", + "version": "0.3.0", "description": "BLOCKSET on JavaScript", "keywords": ["blockset", "content-addressable", "storage", "cdt", "content-dependent-tree", "hash"], "main": "index.mjs", diff --git a/test.mjs b/test.mjs index 15ac7b7..c9f11e9 100644 --- a/test.mjs +++ b/test.mjs @@ -4,17 +4,19 @@ import digest256 from './digest256.mjs' import subtree from './subtree.mjs' import tree from './tree.mjs' import index from './index.mjs' +import ioNode from './io/node.mjs' import fs from 'node:fs' import fsPromises from 'node:fs/promises' /** @typedef {import('./subtree.mjs').State} StateSubTree */ /** @typedef {import('./tree.mjs').State} StateTree */ -/** @typedef {import('./index.mjs').Provider} Provider */ +/** @typedef {import('./io/io.mjs').IO} IO */ const { toAddress, getParityBit } = base32 const { compress } = sha224 const { merge, byteToDigest, len } = digest256 const { highestOne256, height, push: pushSubTree } = subtree const { push: pushTree, end: endTree } = tree -const { get, syncFileProvider, asyncFileProvider, fetchProvider } = index +const { getLocal, getRemote } = index +const { node, nodeSync } = ioNode console.log(`test start`) @@ -231,9 +233,11 @@ const runTest = async (f) => { console.log(`Call to ${f.name} took ${t1 - t0} milliseconds.`); } -/** @type {(provider: Provider) => Promise} */ -const runTestsGet = async (provider) => { - const getWithProvider = get(provider) +/** @typedef {(io: IO) => (root: [string, string]) => Promise} GetFunc*/ + +/** @type {(io: IO) => (getFunc: GetFunc) => Promise} */ +const runTestsGet = io => async (getFunc) => { + const getWithProvider = getFunc(io) const testGet1 = async () => { const exitCode = await getWithProvider(['vqra44skpkefw4bq9k96xt9ks84221dmk1pzaym86cqd6', '_out_list1_async']) if (exitCode !== 0) { throw exitCode } @@ -266,15 +270,13 @@ const runTestsGet = async (provider) => { await runTest(testGetRepeat) } -const testFetchProvider = fetchProvider('410f5a49.blockset-js-test.pages.dev') - const mainTestAsync = async () => { console.log('sync provider') - await runTestsGet(syncFileProvider) + await runTestsGet(nodeSync)(getLocal) console.log('async provider') - await runTestsGet(asyncFileProvider) + await runTestsGet(node)(getLocal) console.log('fetch provider') - await runTestsGet(testFetchProvider) + await runTestsGet(node)(getRemote('410f5a49.blockset-js-test.pages.dev')) } mainTestAsync() \ No newline at end of file diff --git a/web-test.mjs b/web-test.mjs index a1236ed..0df85cf 100644 --- a/web-test.mjs +++ b/web-test.mjs @@ -1,32 +1,37 @@ import getModule from './get.mjs' +import ioWeb from './io/web.mjs' +import index from './index.mjs' /** @typedef {import('./get.mjs').Address} Address */ -const { get, fetchRead } = getModule +const { get } = getModule +const { web } = ioWeb +const { fetchRead } = index +const d = web.document // @ts-ignore -document.getElementById('download').addEventListener('click', () => { +d.getElementById('download').addEventListener('click', () => { reset() // @ts-ignore - const hash = document.getElementById('input-hash').value + const hash = d.getElementById('input-hash').value // @ts-ignore - const host = document.getElementById('input-host').value + const host = d.getElementById('input-host').value let buffer = new Uint8Array() - const fRead = fetchRead(host) + const fRead = fetchRead(host)(web) /** @type {(address: Address) => Promise} */ const read = address => { // @ts-ignore - document.getElementById('log').innerText += `read from ${address}\n` + d.getElementById('log').innerText += `read from ${address}\n` return fRead(address) } /** @type {(b: Uint8Array) => Promise} */ const write = async (b) => { // @ts-ignore - document.getElementById('log').innerText += `write ${b.length}\n` + d.getElementById('log').innerText += `write ${b.length}\n` buffer = new Uint8Array([...buffer, ...b]) } get({ read, write })(hash).then(exitCode => { if (exitCode !== null) { // @ts-ignore - document.getElementById('log').innerText += `error exit code = ${exitCode}\n` + d.getElementById('log').innerText += `error exit code = ${exitCode}\n` return } @@ -34,22 +39,22 @@ document.getElementById('download').addEventListener('click', () => { const image = new Blob([buffer], { type: 'image/jpeg' }); const imageUrl = URL.createObjectURL(image); // @ts-ignore - document.getElementById('output-image').style.display = 'block' + d.getElementById('output-image').style.display = 'block' // @ts-ignore - document.getElementById('output-image').src = imageUrl; + d.getElementById('output-image').src = imageUrl; return } // @ts-ignore - document.getElementById('output-text').style.display = 'block' + d.getElementById('output-text').style.display = 'block' // @ts-ignore - document.getElementById('output-text').innerText = new TextDecoder().decode(buffer) + d.getElementById('output-text').innerText = new TextDecoder().decode(buffer) }) }); const reset = () => { // @ts-ignore - document.getElementById('output-image').style.display = 'none' + d.getElementById('output-image').style.display = 'none' // @ts-ignore - document.getElementById('output-text').style.display = 'none' + d.getElementById('output-text').style.display = 'none' } \ No newline at end of file