From e04e145e2bf03f5a3b0607a2f1a85174576582ba Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 14:21:45 +1300 Subject: [PATCH 01/62] Refactor tick into class. --- packages/daemon/src/common/tick.ts | 95 ++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 packages/daemon/src/common/tick.ts diff --git a/packages/daemon/src/common/tick.ts b/packages/daemon/src/common/tick.ts new file mode 100644 index 00000000..88fa67d9 --- /dev/null +++ b/packages/daemon/src/common/tick.ts @@ -0,0 +1,95 @@ +import { Event, EventTarget } from 'ts-event-target' +import type { Startable } from '@libp2p/interface' + +export class MethodErrorEvent extends Event<'method:error'> { + readonly error: Error + + constructor (error: Error) { + super('method:error') + + this.error = error + } +} + +type Events = EventTarget<[MethodErrorEvent]> + +interface Method { (...args: any[]): any } + +export class Tick implements Startable { + private readonly interval: number + private readonly methods: Method[] = [] + private readonly events: Events = new EventTarget() + private controller: AbortController = new AbortController() + + constructor (interval: number) { + this.interval = interval + } + + add (method: Method): void { + this.methods.push(method) + } + + async start (): Promise { + void this.loop() + } + + async stop (): Promise { + this.controller.abort() + this.controller = new AbortController() + } + + private get signal (): AbortSignal { + return this.controller.signal + } + + private get isAborted (): boolean { + return this.controller.signal.aborted + } + + private async loop () { + for (;;) { + for (const method of this.methods) { + try { + await method() + } catch (e) { + const error = e instanceof Error ? e : new Error(JSON.stringify(e)) + + this.events.dispatchEvent(new MethodErrorEvent(error)) + } + + if (this.isAborted) { + return true + } + } + + await new Promise(resolve => { + const listener = () => { + if (timeout != null) { + clearTimeout(timeout) + this.signal.removeEventListener('abort', listener) + resolve() + } + } + + const timeout = setTimeout(() => { + this.signal.removeEventListener('abort', listener) + resolve() + }, this.interval) + + this.signal.addEventListener('abort', listener) + }) + + if (this.isAborted) { + return + } + } + } +} + +export const createTick = async (interval: number): Promise => { + const tick = new Tick(interval) + + await tick.start() + + return tick +} From 4a6449450111f680005845b505262f4f40d2e0a9 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 14:22:13 +1300 Subject: [PATCH 02/62] Simplify argv and config parsing. --- packages/daemon/src/common/parse-argv.ts | 36 ++++++++++++++++++++++ packages/daemon/src/common/parse-config.ts | 15 +++++++++ 2 files changed, 51 insertions(+) create mode 100644 packages/daemon/src/common/parse-argv.ts create mode 100644 packages/daemon/src/common/parse-config.ts diff --git a/packages/daemon/src/common/parse-argv.ts b/packages/daemon/src/common/parse-argv.ts new file mode 100644 index 00000000..36745034 --- /dev/null +++ b/packages/daemon/src/common/parse-argv.ts @@ -0,0 +1,36 @@ +import Path from 'path' +import { hideBin } from 'yargs/helpers' +import yargs from 'yargs/yargs' +import { projectPath } from '@/utils.js' + +export default async () => { + const argv = await yargs(hideBin(process.argv)) + .option({ + socket: { + alias: 's', + type: 'string', + default: '/tmp/server.socket' + } + }) + .option({ + key: { + alias: 'k', + type: 'string', + default: Path.join(projectPath, 'config/key.json') + } + }) + .option({ + config: { + alias: 'c', + type: 'string', + default: Path.join(projectPath, 'config/config.json') + } + }) + .parse() + + return { + socket: Path.resolve(argv.socket), + key: Path.resolve(argv.key), + config: Path.resolve(argv.config) + } +} diff --git a/packages/daemon/src/common/parse-config.ts b/packages/daemon/src/common/parse-config.ts new file mode 100644 index 00000000..f19f5f51 --- /dev/null +++ b/packages/daemon/src/common/parse-config.ts @@ -0,0 +1,15 @@ +import fs from 'fs/promises' +import { z } from 'zod' + +export interface Provides extends Record { + config: Record + get (shape: T): z.infer +} + +export default async (path: string) => { + const raw = await fs.readFile(path, { encoding: 'utf8' }) + const config = z.record(z.unknown()).parse(JSON.parse(raw)) + const get = (shape: T): z.infer => shape.parse(config) + + return get +} From 0d665f9a92cad466b71874a227e9244c1c4ccbd9 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 14:22:38 +1300 Subject: [PATCH 03/62] Unify config interface. --- packages/daemon/src/common/interface.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 packages/daemon/src/common/interface.ts diff --git a/packages/daemon/src/common/interface.ts b/packages/daemon/src/common/interface.ts new file mode 100644 index 00000000..532d3021 --- /dev/null +++ b/packages/daemon/src/common/interface.ts @@ -0,0 +1,17 @@ +import { z } from 'zod' + +export const Config = z.object({ + storage: z.string().default(':memory:'), + slots: z.number().int().min(1).max(100).default(20), + tickInterval: z.number().default(10 * 60), + serverMode: z.boolean().default(false), + private: z.boolean().default(false), + bootstrap: z.array(z.string()).default([]), + addresses: z.array(z.string()).default([ + '/ip4/127.0.0.1/tcp/0', + '/ip4/127.0.0.1/tcp/0/ws' + ]) +}) + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export type Config = z.output From 96f96139a1f2920f6fed476f54b56a401ca1846c Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 14:40:01 +1300 Subject: [PATCH 04/62] Make stop wait for the loop promise to end in tick. --- packages/daemon/src/common/tick.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/daemon/src/common/tick.ts b/packages/daemon/src/common/tick.ts index 88fa67d9..211670ad 100644 --- a/packages/daemon/src/common/tick.ts +++ b/packages/daemon/src/common/tick.ts @@ -20,6 +20,7 @@ export class Tick implements Startable { private readonly methods: Method[] = [] private readonly events: Events = new EventTarget() private controller: AbortController = new AbortController() + private loopPromise: Promise | null = null constructor (interval: number) { this.interval = interval @@ -30,12 +31,15 @@ export class Tick implements Startable { } async start (): Promise { - void this.loop() + await this.loopPromise + this.controller = new AbortController() + this.loopPromise = this.loop() } async stop (): Promise { this.controller.abort() - this.controller = new AbortController() + + await this.loopPromise } private get signal (): AbortSignal { @@ -46,7 +50,7 @@ export class Tick implements Startable { return this.controller.signal.aborted } - private async loop () { + private async loop (): Promise { for (;;) { for (const method of this.methods) { try { @@ -58,7 +62,7 @@ export class Tick implements Startable { } if (this.isAborted) { - return true + return } } From e9d58e628d231a550b2e4b26c014bd0c928221f2 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 14:40:52 +1300 Subject: [PATCH 05/62] Pass signal to method in tick. --- packages/daemon/src/common/tick.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/daemon/src/common/tick.ts b/packages/daemon/src/common/tick.ts index 211670ad..aaea7745 100644 --- a/packages/daemon/src/common/tick.ts +++ b/packages/daemon/src/common/tick.ts @@ -13,7 +13,7 @@ export class MethodErrorEvent extends Event<'method:error'> { type Events = EventTarget<[MethodErrorEvent]> -interface Method { (...args: any[]): any } +interface Method { (signal?: AbortSignal): any } export class Tick implements Startable { private readonly interval: number @@ -54,7 +54,7 @@ export class Tick implements Startable { for (;;) { for (const method of this.methods) { try { - await method() + await method(this.signal) } catch (e) { const error = e instanceof Error ? e : new Error(JSON.stringify(e)) From 0f4703ec92ddc1d6dee80ef695caa0ceded0a0e2 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 14:53:52 +1300 Subject: [PATCH 06/62] Make events public. --- packages/daemon/src/common/tick.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/daemon/src/common/tick.ts b/packages/daemon/src/common/tick.ts index aaea7745..d41a45b8 100644 --- a/packages/daemon/src/common/tick.ts +++ b/packages/daemon/src/common/tick.ts @@ -18,9 +18,9 @@ interface Method { (signal?: AbortSignal): any } export class Tick implements Startable { private readonly interval: number private readonly methods: Method[] = [] - private readonly events: Events = new EventTarget() private controller: AbortController = new AbortController() private loopPromise: Promise | null = null + readonly events: Events = new EventTarget() constructor (interval: number) { this.interval = interval From cd6954042cebaad1d7bffb69d4b3fab2b1264f21 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 15:07:40 +1300 Subject: [PATCH 07/62] Split the pin manger from the downloader. --- .../src/common/downloader/downloader.ts | 122 +++++++++++++++ .../daemon/src/common/downloader/utils.ts | 7 + .../daemon/src/common/pin-manager/index.ts | 141 ++++++++++++++++++ .../src/common/pin-manager/interface.ts | 15 ++ .../daemon/src/common/pin-manager/utils.js | 30 ++++ packages/daemon/src/common/tick.ts | 4 +- 6 files changed, 316 insertions(+), 3 deletions(-) create mode 100644 packages/daemon/src/common/downloader/downloader.ts create mode 100644 packages/daemon/src/common/downloader/utils.ts create mode 100644 packages/daemon/src/common/pin-manager/index.ts create mode 100644 packages/daemon/src/common/pin-manager/interface.ts create mode 100644 packages/daemon/src/common/pin-manager/utils.js diff --git a/packages/daemon/src/common/downloader/downloader.ts b/packages/daemon/src/common/downloader/downloader.ts new file mode 100644 index 00000000..bab0070e --- /dev/null +++ b/packages/daemon/src/common/downloader/downloader.ts @@ -0,0 +1,122 @@ +import parallel from 'it-parallel' +import { pipe } from 'it-pipe' +import { collect } from 'streaming-iterables' +import { Event, EventTarget } from 'ts-event-target' +import { linearWeightTranslation } from './utils.js' +import type { PinManager } from '../pin-manager/index.js' +import type { Startable } from '@libp2p/interface' +import type { CID } from 'multiformats/cid' + +export class DownloadErrorEvent extends Event<'download:error'> { + readonly error: Error + + constructor (error: Error) { + super('download:error') + + this.error = error + } +} + +export class Downloader implements Startable { + private readonly slots: number + private readonly pinManager: PinManager + private controller: AbortController = new AbortController() + private loopPromise: Promise | null = null + readonly events = new EventTarget<[DownloadErrorEvent]>() + + constructor (pinManager: PinManager, slots: number) { + this.slots = slots + this.pinManager = pinManager + } + + async start () { + await this.loopPromise + this.controller = new AbortController() + this.loopPromise = this.loop() + } + + async stop () { + this.controller.abort() + await this.loopPromise + } + + private async loop () { + for (;;) { + if (this.isAborted) { + return + } + + await pipe( + () => this.delay(), + i => this.getPins(i), + i => this.batchDownload(i), + i => parallel(i, { concurrency: this.slots, ordered: false }), + i => this.catcher(i), + async i => collect(i) + ) + + await new Promise(resolve => setTimeout(resolve, 100)) + } + } + + private async * batchDownload (itr: AsyncIterable<[CID, number]>): AsyncGenerator<() => Promise<{ cid: CID, block: Uint8Array }>, void, undefined> { + for await (const [cid, priority] of itr) { + if (this.isAborted) { + return + } + + const weight = Math.floor(linearWeightTranslation(priority / 100) * this.slots) + 1 + const downloaders = await this.pinManager.download(cid, { limit: weight }) + + yield * downloaders + } + } + + private async * catcher (itr: AsyncIterable): AsyncIterable { + try { + yield * itr + } catch (e) { + const error = e instanceof Error ? e : new Error(JSON.stringify(e)) + + this.events.dispatchEvent(new DownloadErrorEvent(error)) + + yield * this.catcher(itr) + } + } + + private async * delay (): AsyncGenerator { + for (;;) { + if (this.isAborted) { + return + } + + await new Promise(resolve => setTimeout(resolve, 100)) + yield + } + } + + async * getPins (loop: AsyncIterable): AsyncGenerator<[CID, number]> { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _ of loop) { + for await (const { value } of this.pinManager.getActive()) { + if (this.isAborted) { + return + } + + yield [value.cid, value.priority] + } + } + } + + private get isAborted (): boolean { + return this.controller.signal.aborted + } +} + +export const createDownloader = async (pinManage: PinManager, slots: number): Promise => { + const downloader = new Downloader(pinManage, slots) + + await downloader.start() + + return downloader +} diff --git a/packages/daemon/src/common/downloader/utils.ts b/packages/daemon/src/common/downloader/utils.ts new file mode 100644 index 00000000..2b77bf57 --- /dev/null +++ b/packages/daemon/src/common/downloader/utils.ts @@ -0,0 +1,7 @@ +export const linearWeightTranslation = (p: number): number => { + return 1 - p +} + +export const logWeightTranslation = (p: number): number => { + return 1 - Math.log10((10 - 1) * p - 1) +} diff --git a/packages/daemon/src/common/pin-manager/index.ts b/packages/daemon/src/common/pin-manager/index.ts new file mode 100644 index 00000000..42f470eb --- /dev/null +++ b/packages/daemon/src/common/pin-manager/index.ts @@ -0,0 +1,141 @@ +import { Key, type Datastore } from 'interface-datastore' +import all from 'it-all' +import { Event, EventTarget } from 'ts-event-target' +import { encodePinInfo, decodePinInfo } from './utils.js' +import type { PinInfo } from './interface.js' +import type { Pair } from '@/interface.js' +import type { BlockInfo } from '@organicdesign/db-helia-pin-manager' +import type HeliaPinManager from '@organicdesign/db-helia-pin-manager' +import type { CID } from 'multiformats/cid' + +type EventTypes = 'reference:removed' + +export class PinManagerEvent extends Event { + readonly key: string + readonly priority: number + readonly cid: CID + + constructor (type: EventTypes, data: { key: string } & PinInfo) { + super(type) + + this.key = data.key + this.cid = data.cid + this.priority = data.priority + } +} + +export class PinManager { + private readonly datastore: Datastore + private readonly pinManager: HeliaPinManager + readonly events = new EventTarget<[PinManagerEvent]>() + + constructor (components: { datastore: Datastore, pinManager: HeliaPinManager }) { + this.datastore = components.datastore + this.pinManager = components.pinManager + } + + async put (key: string, pinInfo: PinInfo): Promise { + const data = encodePinInfo(pinInfo) + + // Need to ensure that the references get updated. + await this.remove(key) + + await this.pinManager.pin(pinInfo.cid) + + await this.datastore.put(new Key(key), data) + } + + async has (key: string, cid?: CID): Promise { + const pinInfo = await this.get(key) + + if (pinInfo == null) { + return false + } + + if (cid == null) { + return true + } + + return pinInfo.cid.equals(cid) + } + + async * getActive (): AsyncGenerator> { + for (const pin of await this.pinManager.getActiveDownloads()) { + yield * this.getByPin(pin) + } + } + + async download (pin: CID, options?: { limit: number }): Promise Promise>> { + return this.pinManager.downloadSync(pin, options) + } + + async getState (cid: CID): Promise<'COMPLETED' | 'DOWNLOADING' | 'DESTROYED' | 'UPLOADING' | 'NOTFOUND'> { + return this.pinManager.getState(cid) + } + + async getSpeed (cid: CID, range?: number): Promise { + return this.pinManager.getSpeed(cid, range) + } + + async getSize (cid: CID): Promise { + return this.pinManager.getSize(cid) + } + + async getBlockCount (cid: CID): Promise { + return this.pinManager.getBlockCount(cid) + } + + async remove (key: string): Promise { + const pinInfo = await this.get(key) + + if (pinInfo == null) { + return + } + + const keys = await all(this.getByPin(pinInfo.cid)) + + // If we only have 1 reference be sure to unpin it. + if (keys.length <= 1) { + await this.pinManager.unpin(pinInfo.cid) + } + + this.events.dispatchEvent(new PinManagerEvent('reference:removed', { key, ...pinInfo })) + + await this.datastore.delete(new Key(key)) + } + + async get (key: string): Promise { + try { + const data = await this.datastore.get(new Key(key)) + + return decodePinInfo(data) + } catch (error) { + return null + } + } + + private async * getByPin (pin: CID): AsyncGenerator> { + const itr = this.datastore.query({ + filters: [({ value }) => { + const pinInfo = decodePinInfo(value) + + if (pinInfo == null) { + return false + } + + return pinInfo.cid.equals(pin) + }] + }) + + for await (const pair of itr) { + const pinInfo = decodePinInfo(pair.value) + + if (pinInfo == null) { + await this.datastore.delete(pair.key) + continue + } + + yield { key: pair.key.toString(), value: pinInfo } + } + } +} diff --git a/packages/daemon/src/common/pin-manager/interface.ts b/packages/daemon/src/common/pin-manager/interface.ts new file mode 100644 index 00000000..27546291 --- /dev/null +++ b/packages/daemon/src/common/pin-manager/interface.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' +import type { CID } from 'multiformats/cid' + +export const EncodedPinInfo = z.object({ + priority: z.number().int().min(1).max(100).optional().default(100), + cid: z.instanceof(Uint8Array) +}) + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export type EncodedPinInfo = z.output + +export interface PinInfo { + priority: number + cid: CID +} diff --git a/packages/daemon/src/common/pin-manager/utils.js b/packages/daemon/src/common/pin-manager/utils.js new file mode 100644 index 00000000..71f75b47 --- /dev/null +++ b/packages/daemon/src/common/pin-manager/utils.js @@ -0,0 +1,30 @@ +import { CID } from 'multiformats/cid' +import { EncodedPinInfo, type PinInfo } from './interface.js' +import { decodeAny, encodeAny } from '@/utils.js' + +export const decodePinInfo = (data: Uint8Array): PinInfo | null => { + const obj = decodeAny(data) + + if (obj == null) { + return null + } + + const encodedPinInfo = EncodedPinInfo.parse(obj) + + return { + ...encodedPinInfo, + cid: CID.decode(encodedPinInfo.cid) + } +} + +export const encodePinInfo = (pinInfo: PinInfo): Uint8Array => { + const encodedPinInfo: EncodedPinInfo = { + ...pinInfo, + cid: pinInfo.cid.bytes + } + + // This will strip foreign keys. + const parsedEncodedPinInfo = EncodedPinInfo.parse(encodedPinInfo) + + return encodeAny(parsedEncodedPinInfo) +} diff --git a/packages/daemon/src/common/tick.ts b/packages/daemon/src/common/tick.ts index d41a45b8..7041028e 100644 --- a/packages/daemon/src/common/tick.ts +++ b/packages/daemon/src/common/tick.ts @@ -11,8 +11,6 @@ export class MethodErrorEvent extends Event<'method:error'> { } } -type Events = EventTarget<[MethodErrorEvent]> - interface Method { (signal?: AbortSignal): any } export class Tick implements Startable { @@ -20,7 +18,7 @@ export class Tick implements Startable { private readonly methods: Method[] = [] private controller: AbortController = new AbortController() private loopPromise: Promise | null = null - readonly events: Events = new EventTarget() + readonly events = new EventTarget<[MethodErrorEvent]>() constructor (interval: number) { this.interval = interval From df214a091069b8cea789ca1d500772f97fbfdb65 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 15:13:18 +1300 Subject: [PATCH 08/62] Rename downloader. --- packages/daemon/src/common/downloader/{downloader.ts => index.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/daemon/src/common/downloader/{downloader.ts => index.ts} (100%) diff --git a/packages/daemon/src/common/downloader/downloader.ts b/packages/daemon/src/common/downloader/index.ts similarity index 100% rename from packages/daemon/src/common/downloader/downloader.ts rename to packages/daemon/src/common/downloader/index.ts From 259d76d47bec2038974d5fa56c99fd9fbd87dec8 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 15:14:39 +1300 Subject: [PATCH 09/62] Setup modules. --- packages/daemon/src/common/index.ts | 108 +++++++++++++++++++++++++++ packages/daemon/src/common/libp2p.ts | 73 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 packages/daemon/src/common/index.ts create mode 100644 packages/daemon/src/common/libp2p.ts diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts new file mode 100644 index 00000000..331ace1b --- /dev/null +++ b/packages/daemon/src/common/index.ts @@ -0,0 +1,108 @@ +import Path from 'path' +import { bitswap } from '@helia/block-brokers' +import HeliaPinManager from '@organicdesign/db-helia-pin-manager' +import { createKeyManager } from '@organicdesign/db-key-manager' +import { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' +import { MemoryBlockstore } from 'blockstore-core' +import { FsBlockstore } from 'blockstore-fs' +import { MemoryDatastore } from 'datastore-core' +import { FsDatastore } from 'datastore-fs' +import { createHelia } from 'helia' +import { createDownloader } from './downloader/index.js' +import { Config } from './interface.js' +import createLibp2p from './libp2p.js' +import parseArgv from './parse-argv.js' +import parseConfig from './parse-config.js' +import { PinManager } from './pin-manager/index.js' +import { createTick } from './tick.js' +import { createLogger } from '@/logger.js' +import { isMemory, extendDatastore } from '@/utils.js' + +export default async () => { + const argv = await parseArgv() + const logger = createLogger('common') + const getConfig = await parseConfig(argv.socket) + const keyManager = await createKeyManager(argv.key) + const controller = new AbortController() + + const config = getConfig(Config) + + const datastore = isMemory(config.storage) + ? new MemoryDatastore() + : new FsDatastore(Path.join(config.storage, 'datastore')) + + const blockstore = isMemory(config.storage) + ? new MemoryBlockstore() + : new FsBlockstore(Path.join(config.storage, 'blockstore')) + + const peerId = await keyManager.getPeerId() + const psk = keyManager.getPskKey() + + const libp2p = await createLibp2p({ + datastore: extendDatastore(datastore, 'datastore/libp2p'), + psk: config.private ? psk : undefined, + peerId, + ...config + }) + + const manualBlockBroker = new ManualBlockBroker() + + const helia = await createHelia({ + datastore: extendDatastore(datastore, 'helia/datastore'), + libp2p, + blockstore, + blockBrokers: [bitswap(), () => manualBlockBroker] + }) + + const heliaPinManager = new HeliaPinManager({ + helia, + datastore: extendDatastore(datastore, 'pinManager') + }) + + heliaPinManager.events.addEventListener('downloads:added', ({ cid }) => { + logger.info(`[downloads] [+] ${cid}`) + }) + + heliaPinManager.events.addEventListener('pins:added', ({ cid }) => { + logger.info(`[pins] [+] ${cid}`) + }) + + heliaPinManager.events.addEventListener('pins:adding', ({ cid }) => { + logger.info(`[pins] [~] ${cid}`) + }) + + heliaPinManager.events.addEventListener('pins:removed', ({ cid }) => { + logger.info(`[pins] [-] ${cid}`) + }) + + const tick = await createTick(config.tickInterval) + + tick.events.addEventListener('method:error', ({ error }) => { + logger.error('tick: ', error) + }) + + const pinManager = new PinManager({ + pinManager: heliaPinManager, + datastore + }) + + pinManager.events.addEventListener('reference:removed', ({ key }) => { + logger.info(`[references] [-] ${key}`) + }) + + const downloader = await createDownloader(pinManager, config.slots) + + downloader.events.addEventListener('download:error', ({ error }) => { + logger.error('downloader: ', error) + }) + + controller.signal.addEventListener('abort', () => { + (async () => { + await tick.stop() + await helia.stop() + await libp2p.stop() + })().catch(error => { + logger.error(error) + }) + }) +} diff --git a/packages/daemon/src/common/libp2p.ts b/packages/daemon/src/common/libp2p.ts new file mode 100644 index 00000000..5b472891 --- /dev/null +++ b/packages/daemon/src/common/libp2p.ts @@ -0,0 +1,73 @@ +import { gossipsub } from '@chainsafe/libp2p-gossipsub' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { autoNAT } from '@libp2p/autonat' +import { bootstrap } from '@libp2p/bootstrap' +import { circuitRelayTransport, circuitRelayServer } from '@libp2p/circuit-relay-v2' +import { dcutr } from '@libp2p/dcutr' +import { identify } from '@libp2p/identify' +import { kadDHT, removePrivateAddressesMapper, removePublicAddressesMapper } from '@libp2p/kad-dht' +import { preSharedKey } from '@libp2p/pnet' +import { tcp } from '@libp2p/tcp' +import { uPnPNAT } from '@libp2p/upnp-nat' +import { webSockets } from '@libp2p/websockets' +import { createLibp2p } from 'libp2p' +import type { Libp2p, PeerId } from '@libp2p/interface' +import type { Datastore } from 'interface-datastore' + +export default async ({ datastore, peerId, psk, addresses, bootstrap: bs, serverMode }: { datastore?: Datastore, peerId?: PeerId, psk?: Uint8Array, addresses?: string[], bootstrap?: string[], serverMode?: boolean }): Promise => { + const services: Record = {} + + if (serverMode === true) { + services.circuitRelay = circuitRelayServer() + } + + return createLibp2p({ + peerId, + datastore, + transports: [tcp(), webSockets(), circuitRelayTransport({ discoverRelays: 2 })], + connectionEncryption: [noise()], + streamMuxers: [yamux()], + connectionProtector: (psk != null) ? preSharedKey({ psk }) : undefined, + + addresses: { + listen: addresses ?? [ + '/ip4/127.0.0.1/tcp/0', + '/ip4/127.0.0.1/tcp/0/ws' + // "/ip6/::/tcp/0" + ] + }, + + connectionManager: { + autoDialInterval: 6e3 + }, + + services: { + ...services, + identify: identify(), + pubsub: gossipsub({ allowPublishToZeroPeers: true }), + autoNAT: autoNAT(), + dcutr: dcutr(), + upnpNAT: uPnPNAT(), + + dht: kadDHT({ + protocol: '/ipfs/kad/1.0.0', + peerInfoMapper: removePrivateAddressesMapper + }), + + lanDHT: kadDHT({ + protocol: '/ipfs/lan/kad/1.0.0', + peerInfoMapper: removePublicAddressesMapper, + clientMode: false + }) + }, + + peerDiscovery: (bs != null) && bs.length > 0 + ? [ + bootstrap({ + list: bs ?? [] + }) + ] + : [] + }) +} From d221e5d699c4af426cb857bc41e59a8ac8dceb37 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 15:16:55 +1300 Subject: [PATCH 10/62] Add welo. --- packages/daemon/src/common/index.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 331ace1b..dd8fdcec 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -8,6 +8,7 @@ import { FsBlockstore } from 'blockstore-fs' import { MemoryDatastore } from 'datastore-core' import { FsDatastore } from 'datastore-fs' import { createHelia } from 'helia' +import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' import { createDownloader } from './downloader/index.js' import { Config } from './interface.js' import createLibp2p from './libp2p.js' @@ -54,6 +55,13 @@ export default async () => { blockBrokers: [bitswap(), () => manualBlockBroker] }) + const welo = await createWelo({ + // @ts-expect-error Helia version mismatch here. + ipfs: network.helia, + replicators: [bootstrapReplicator(), pubsubReplicator()], + identity: await keyManager.getWeloIdentity() + }) + const heliaPinManager = new HeliaPinManager({ helia, datastore: extendDatastore(datastore, 'pinManager') @@ -99,6 +107,8 @@ export default async () => { controller.signal.addEventListener('abort', () => { (async () => { await tick.stop() + await welo.stop() + await downloader.stop() await helia.stop() await libp2p.stop() })().catch(error => { From 77ec585f6dadff31c9d023ca4aa2d05c4060249d Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 15:29:11 +1300 Subject: [PATCH 11/62] Add groups. --- packages/daemon/src/common/groups.ts | 97 ++++++++++++++++++++++++++++ packages/daemon/src/common/index.ts | 13 +++- 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 packages/daemon/src/common/groups.ts diff --git a/packages/daemon/src/common/groups.ts b/packages/daemon/src/common/groups.ts new file mode 100644 index 00000000..0b95e682 --- /dev/null +++ b/packages/daemon/src/common/groups.ts @@ -0,0 +1,97 @@ +import { Key } from 'interface-datastore' +import { Event, EventTarget } from 'ts-event-target' +import { Manifest } from 'welo/manifest/index' +import { decodeCbor } from 'welo/utils/block' +import type { Pair, KeyvalueDB } from '@/interface.js' +import type { Startable } from '@libp2p/interfaces/startable' +import type { Datastore } from 'interface-datastore' +import type { CID } from 'multiformats/cid' +import type { Welo } from 'welo' +import type { ManifestData } from 'welo/manifest/interface' + +export interface Components { + datastore: Datastore + welo: Welo +} + +type EventTypes = 'groups:joined' + +export class GroupEvent extends Event { + readonly cid: CID + + constructor (type: EventTypes, cid: CID) { + super(type) + + this.cid = cid + } +} + +export class Groups implements Startable { + private readonly welo: Welo + private readonly datastore: Datastore + private readonly groups = new Map() + private started = false + readonly events = new EventTarget<[GroupEvent]>() + + constructor (components: Components) { + this.welo = components.welo + this.datastore = components.datastore + } + + isStarted (): boolean { + return this.started + } + + async start (): Promise { + if (this.started) { + return + } + + for await (const pair of this.datastore.query({})) { + const block = await decodeCbor(pair.value) + const manifest = Manifest.asManifest({ block }) + + if (manifest == null) { + continue + } + + await this.add(manifest) + } + + this.started = true + } + + async stop (): Promise { + this.groups.clear() + this.started = false + } + + async add (manifest: Manifest): Promise { + const database = await this.welo.open(manifest) as KeyvalueDB + const address = manifest.address.cid.toString() + + this.groups.set(address, database) + + await this.datastore.put(new Key(database.address.cid.toString()), database.manifest.block.bytes) + + this.events.dispatchEvent(new GroupEvent('groups:joined', manifest.address.cid)) + } + + get (group: CID): KeyvalueDB | undefined { + return this.groups.get(group.toString()) + } + + * all (): Iterable> { + for (const [key, value] of this.groups.entries()) { + yield { key, value } + } + } +} + +export const createGroups = async (components: Components): Promise => { + const groups = new Groups(components) + + await groups.start() + + return groups +} diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index dd8fdcec..3aaa2789 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -10,6 +10,7 @@ import { FsDatastore } from 'datastore-fs' import { createHelia } from 'helia' import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' import { createDownloader } from './downloader/index.js' +import { createGroups } from './groups.js' import { Config } from './interface.js' import createLibp2p from './libp2p.js' import parseArgv from './parse-argv.js' @@ -62,6 +63,15 @@ export default async () => { identity: await keyManager.getWeloIdentity() }) + const groups = await createGroups({ + datastore: extendDatastore(datastore, 'groups'), + welo + }) + + groups.events.addEventListener('groups:joined', ({ cid }) => { + logger.info(`[groups] [join] ${cid.toString()}`) + }) + const heliaPinManager = new HeliaPinManager({ helia, datastore: extendDatastore(datastore, 'pinManager') @@ -107,8 +117,9 @@ export default async () => { controller.signal.addEventListener('abort', () => { (async () => { await tick.stop() - await welo.stop() await downloader.stop() + await groups.stop() + await welo.stop() await helia.stop() await libp2p.stop() })().catch(error => { From 3d5fefcb733140df66e7654e9ac2f9a277e03e0a Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 15:39:53 +1300 Subject: [PATCH 12/62] Add sneakernet. --- packages/daemon/src/common/index.ts | 18 +++ .../daemon/src/common/sneakernet/index.ts | 145 ++++++++++++++++++ .../daemon/src/common/sneakernet/interface.ts | 22 +++ 3 files changed, 185 insertions(+) create mode 100644 packages/daemon/src/common/sneakernet/index.ts create mode 100644 packages/daemon/src/common/sneakernet/interface.ts diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 3aaa2789..bb7b5d73 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -10,13 +10,16 @@ import { FsDatastore } from 'datastore-fs' import { createHelia } from 'helia' import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' import { createDownloader } from './downloader/index.js' +import { EntryTracker } from './entry-tracker.js' import { createGroups } from './groups.js' import { Config } from './interface.js' import createLibp2p from './libp2p.js' import parseArgv from './parse-argv.js' import parseConfig from './parse-config.js' import { PinManager } from './pin-manager/index.js' +import { Sneakernet } from './sneakernet/index.js' import { createTick } from './tick.js' +import type { KeyvalueDB } from '@/interface.js' import { createLogger } from '@/logger.js' import { isMemory, extendDatastore } from '@/utils.js' @@ -72,6 +75,16 @@ export default async () => { logger.info(`[groups] [join] ${cid.toString()}`) }) + const sneakernet = new Sneakernet({ + welo, + helia, + datastore: extendDatastore(datastore, 'sneakernet'), + manualBlockBroker, + groups + }) + + const getTracker = (database: KeyvalueDB): EntryTracker => new EntryTracker(database) + const heliaPinManager = new HeliaPinManager({ helia, datastore: extendDatastore(datastore, 'pinManager') @@ -126,4 +139,9 @@ export default async () => { logger.error(error) }) }) + + return { + sneakernet, + getTracker + } } diff --git a/packages/daemon/src/common/sneakernet/index.ts b/packages/daemon/src/common/sneakernet/index.ts new file mode 100644 index 00000000..076680bb --- /dev/null +++ b/packages/daemon/src/common/sneakernet/index.ts @@ -0,0 +1,145 @@ +import fss from 'fs' +import fs from 'fs/promises' +import Path from 'path' +import { Readable } from 'stream' +import { car, type Car } from '@helia/car' +import { CarWriter, CarReader } from '@ipld/car' +import * as cborg from 'cborg' +import { type Datastore, Key } from 'interface-datastore' +import all from 'it-all' +import { CID } from 'multiformats/cid' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { getHeads } from 'welo/utils/replicator' +import { EncodedPeerData, PeerData } from './interface.js' +import type { Groups } from '../groups.js' +import type { KeyvalueDB, Pair } from '@/interface.js' +import type { Helia } from '@helia/interface' +import type { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' +import type { Blockstore } from 'interface-blockstore' +import type { Welo } from 'welo' + +export interface Components { + helia: Helia + welo: Welo + datastore: Datastore + manualBlockBroker: ManualBlockBroker + groups: Groups +} + +export class Sneakernet { + private readonly datastore: Datastore + private readonly blockstore: Blockstore + private readonly id: Uint8Array + private readonly broker: ManualBlockBroker + private readonly getGroups: () => Array> + private readonly car: Car + + constructor (components: Components) { + this.datastore = components.datastore + this.id = components.welo.identity.id + this.broker = components.manualBlockBroker + this.getGroups = () => [...components.groups.all()] + this.car = car(components.helia) + this.blockstore = components.helia.blockstore + } + + async export (path: string, peers: string[] = []): Promise { + await Promise.all([ + this.exportManifest(Path.join(path, 'manifest')), + this.exportBlocks(Path.join(path, 'blocks.car'), peers) + ]) + } + + async import (path: string): Promise { + await Promise.all([ + this.importManifest(Path.join(path, 'manifest')), + this.importBlocks(Path.join(path, 'blocks.car')) + ]) + } + + private async importManifest (path: string): Promise { + const rawData = await fs.readFile(path) + const data = EncodedPeerData.parse(cborg.decode(rawData)) + + await all(this.datastore.putMany(data.map(peerData => ({ + key: new Key(peerData.id), + value: cborg.encode({ + wants: peerData.wants, + heads: peerData.heads + }) + })))) + } + + private async importBlocks (path: string): Promise { + try { + await fs.stat(path) + } catch (error) { + // No blocks file - just return. + return + } + + const inStream = fss.createReadStream(path) + const reader = await CarReader.fromIterable(inStream) + + await this.car.import(reader) + } + + private async exportManifest (path: string): Promise { + const pairs = await Promise.all( + this.getGroups().map(async ({ value: database }) => ({ + group: database.address.cid.bytes, + cids: (await getHeads(database.replica)).map(h => h.bytes) + })) + ) + + const peerData: EncodedPeerData = [{ + id: this.id, + wants: [...this.broker.getWants()].map(c => c.bytes), + heads: pairs + }] + + const data = cborg.encode(peerData) + + await fs.writeFile(path, data) + } + + private async exportBlocks (path: string, peers: string[] = []): Promise { + if (peers.length === 0) { + return + } + + const blocks: CID[] = [] + + const addWants = async (wants: Uint8Array[]): Promise => { + for (const cidData of wants) { + const cid = CID.decode(cidData) + + if (await this.blockstore.has(cid)) { + blocks.push(cid) + } + } + } + + for (const peer of peers) { + try { + const key = new Key(uint8ArrayFromString(peer, 'base58btc')) + const data = await this.datastore.get(key) + const peerData = PeerData.parse(cborg.decode(data)) + + await addWants(peerData.wants) + } catch (error) { + // Ignore + } + } + + if (blocks.length === 0) { + return + } + + const { writer, out } = await CarWriter.create(blocks) + + Readable.from(out).pipe(fss.createWriteStream(path)) + + await this.car.export(blocks, writer) + } +} diff --git a/packages/daemon/src/common/sneakernet/interface.ts b/packages/daemon/src/common/sneakernet/interface.ts new file mode 100644 index 00000000..69061fd2 --- /dev/null +++ b/packages/daemon/src/common/sneakernet/interface.ts @@ -0,0 +1,22 @@ +import { z } from 'zod' + +export const PeerData = z.object({ + // This is a list of blocks that this peer wants. + wants: z.array(z.instanceof(Uint8Array)), + + // This is a list of the heads this peer is known to have. + heads: z.array(z.object({ + group: z.instanceof(Uint8Array), + cids: z.array(z.instanceof(Uint8Array)) + })) +}) + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export type PeerData = z.infer + +export const EncodedPeerData = z.array(z.object({ + id: z.instanceof(Uint8Array) +}).extend(PeerData.shape)) + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export type EncodedPeerData = z.infer From abca7c1a7c442e645051ea5b11216cdc160d2591 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 15:40:10 +1300 Subject: [PATCH 13/62] Add entry tracker. --- packages/daemon/src/common/entry-tracker.ts | 67 +++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 packages/daemon/src/common/entry-tracker.ts diff --git a/packages/daemon/src/common/entry-tracker.ts b/packages/daemon/src/common/entry-tracker.ts new file mode 100644 index 00000000..90f9e8b1 --- /dev/null +++ b/packages/daemon/src/common/entry-tracker.ts @@ -0,0 +1,67 @@ +import { Key, type Datastore } from 'interface-datastore' +import { sha256 } from 'multiformats/hashes/sha2' +import { compare as uint8ArrayCompare } from 'uint8arrays/compare' +import type { KeyvalueDB, Pair } from '@/interface.js' +import { encodeAny, extendDatastore } from '@/utils.js' + +// Get the hash data from raw data. +const hashEntry = async (entry: unknown): Promise => { + const data = encodeAny(entry) + const digest = await sha256.digest(data) + + return digest.bytes +} + +// This class is responsible for keeping track of entries in the database. +export class EntryTracker { + private readonly datastore: Datastore + private readonly database: KeyvalueDB + + constructor (database: KeyvalueDB) { + this.datastore = extendDatastore(database.datastore, 'entry-tracker') + this.database = database + } + + async * process (prefix: string = '/'): AsyncGenerator> { + const index = await this.database.store.latest() + + for await (const { key, value } of index.query({ prefix })) { + const str = key.toString() + + if (await this.validate(str, value)) { + continue + } + + yield { key: key.toString(), value } + + await this.put(str, value) + } + } + + // Process an entry. + async put (key: string, entry: unknown): Promise { + const hash = await hashEntry(entry) + + await this.datastore.put(new Key(key), hash) + } + + async validate (key: string, entry: unknown): Promise { + const eHash = await this.getHash(key) + + if (eHash == null) { + return false + } + + const hash = await hashEntry(entry) + + return uint8ArrayCompare(eHash, hash) === 0 + } + + private async getHash (key: string): Promise { + try { + return await this.datastore.get(new Key(key)) + } catch (error) { + return null + } + } +} From 3a9fdcedde99d0f4ba3a1f949126d93f0929a9bf Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 15:46:21 +1300 Subject: [PATCH 14/62] Add net server. --- packages/daemon/src/common/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index bb7b5d73..99bf2097 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -22,6 +22,7 @@ import { createTick } from './tick.js' import type { KeyvalueDB } from '@/interface.js' import { createLogger } from '@/logger.js' import { isMemory, extendDatastore } from '@/utils.js' +import { createNetServer } from '@organicdesign/net-rpc' export default async () => { const argv = await parseArgv() @@ -29,7 +30,7 @@ export default async () => { const getConfig = await parseConfig(argv.socket) const keyManager = await createKeyManager(argv.key) const controller = new AbortController() - + const net = await createNetServer(argv.socket) const config = getConfig(Config) const datastore = isMemory(config.storage) @@ -129,6 +130,7 @@ export default async () => { controller.signal.addEventListener('abort', () => { (async () => { + await net.close() await tick.stop() await downloader.stop() await groups.stop() From 8ea20f06c2b792b0877e6a30be4799fa2761bb1a Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:01:35 +1300 Subject: [PATCH 15/62] Add components interface. --- packages/daemon/src/common/index.ts | 29 ++++++++++++++++------ packages/daemon/src/common/interface.ts | 28 +++++++++++++++++++++ packages/daemon/src/common/parse-argv.ts | 6 ++++- packages/daemon/src/common/parse-config.ts | 2 +- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 99bf2097..d0be9a34 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -3,6 +3,7 @@ import { bitswap } from '@helia/block-brokers' import HeliaPinManager from '@organicdesign/db-helia-pin-manager' import { createKeyManager } from '@organicdesign/db-key-manager' import { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' +import { createNetServer } from '@organicdesign/net-rpc' import { MemoryBlockstore } from 'blockstore-core' import { FsBlockstore } from 'blockstore-fs' import { MemoryDatastore } from 'datastore-core' @@ -12,7 +13,7 @@ import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' import { createDownloader } from './downloader/index.js' import { EntryTracker } from './entry-tracker.js' import { createGroups } from './groups.js' -import { Config } from './interface.js' +import { Config, type Components } from './interface.js' import createLibp2p from './libp2p.js' import parseArgv from './parse-argv.js' import parseConfig from './parse-config.js' @@ -22,15 +23,14 @@ import { createTick } from './tick.js' import type { KeyvalueDB } from '@/interface.js' import { createLogger } from '@/logger.js' import { isMemory, extendDatastore } from '@/utils.js' -import { createNetServer } from '@organicdesign/net-rpc' -export default async () => { +export default async (): Promise => { const argv = await parseArgv() const logger = createLogger('common') const getConfig = await parseConfig(argv.socket) const keyManager = await createKeyManager(argv.key) const controller = new AbortController() - const net = await createNetServer(argv.socket) + const net = await createNetServer(argv.socket) const config = getConfig(Config) const datastore = isMemory(config.storage) @@ -130,7 +130,7 @@ export default async () => { controller.signal.addEventListener('abort', () => { (async () => { - await net.close() + await net.close() await tick.stop() await downloader.stop() await groups.stop() @@ -142,8 +142,21 @@ export default async () => { }) }) - return { + const components: Components = { sneakernet, - getTracker - } + getTracker, + helia, + libp2p, + blockstore, + datastore, + net, + tick, + downloader, + getConfig, + controller, + groups, + pinManager + } + + return components } diff --git a/packages/daemon/src/common/interface.ts b/packages/daemon/src/common/interface.ts index 532d3021..4ab4b067 100644 --- a/packages/daemon/src/common/interface.ts +++ b/packages/daemon/src/common/interface.ts @@ -1,4 +1,16 @@ import { z } from 'zod' +import type { Downloader } from './downloader/index.js' +import type { EntryTracker } from './entry-tracker.js' +import type { Groups } from './groups.js' +import type { PinManager } from './pin-manager/index.js' +import type { Sneakernet } from './sneakernet/index.js' +import type { Tick } from './tick.js' +import type { KeyvalueDB } from '@/interface.js' +import type { Helia } from '@helia/interface' +import type { Libp2p } from '@libp2p/interface' +import type { NetServer } from '@organicdesign/net-rpc' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' export const Config = z.object({ storage: z.string().default(':memory:'), @@ -15,3 +27,19 @@ export const Config = z.object({ // eslint-disable-next-line @typescript-eslint/no-redeclare export type Config = z.output + +export interface Components { + helia: Helia + libp2p: Libp2p + datastore: Datastore + blockstore: Blockstore + controller: AbortController + net: NetServer + tick: Tick + sneakernet: Sneakernet + getTracker(keyvalueDB: KeyvalueDB): EntryTracker + getConfig(shape: T): z.infer + downloader: Downloader + groups: Groups + pinManager: PinManager +} diff --git a/packages/daemon/src/common/parse-argv.ts b/packages/daemon/src/common/parse-argv.ts index 36745034..83c1f4dc 100644 --- a/packages/daemon/src/common/parse-argv.ts +++ b/packages/daemon/src/common/parse-argv.ts @@ -3,7 +3,11 @@ import { hideBin } from 'yargs/helpers' import yargs from 'yargs/yargs' import { projectPath } from '@/utils.js' -export default async () => { +export default async (): Promise<{ + socket: string + key: string + config: string +}> => { const argv = await yargs(hideBin(process.argv)) .option({ socket: { diff --git a/packages/daemon/src/common/parse-config.ts b/packages/daemon/src/common/parse-config.ts index f19f5f51..4c37e145 100644 --- a/packages/daemon/src/common/parse-config.ts +++ b/packages/daemon/src/common/parse-config.ts @@ -6,7 +6,7 @@ export interface Provides extends Record { get (shape: T): z.infer } -export default async (path: string) => { +export default async (path: string): Promise<(shape: T) => z.infer> => { const raw = await fs.readFile(path, { encoding: 'utf8' }) const config = z.record(z.unknown()).parse(JSON.parse(raw)) const get = (shape: T): z.infer => shape.parse(config) From 611ee8183491dd1815688bce3216c14060b59355 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:12:09 +1300 Subject: [PATCH 16/62] Add welo to components interface. --- packages/daemon/src/common/index.ts | 4 ++-- packages/daemon/src/common/interface.ts | 2 ++ packages/daemon/src/interface.ts | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index d0be9a34..fde5ffa4 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -142,7 +142,7 @@ export default async (): Promise => { }) }) - const components: Components = { + const components: Components = { sneakernet, getTracker, helia, @@ -156,7 +156,7 @@ export default async (): Promise => { controller, groups, pinManager - } + } return components } diff --git a/packages/daemon/src/common/interface.ts b/packages/daemon/src/common/interface.ts index 4ab4b067..d1f1315c 100644 --- a/packages/daemon/src/common/interface.ts +++ b/packages/daemon/src/common/interface.ts @@ -11,6 +11,7 @@ import type { Libp2p } from '@libp2p/interface' import type { NetServer } from '@organicdesign/net-rpc' import type { Blockstore } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' +import type { Welo } from 'welo' export const Config = z.object({ storage: z.string().default(':memory:'), @@ -31,6 +32,7 @@ export type Config = z.output export interface Components { helia: Helia libp2p: Libp2p + welo: Welo datastore: Datastore blockstore: Blockstore controller: AbortController diff --git a/packages/daemon/src/interface.ts b/packages/daemon/src/interface.ts index fa4641fa..1b86b409 100644 --- a/packages/daemon/src/interface.ts +++ b/packages/daemon/src/interface.ts @@ -1,3 +1,4 @@ +import type { Components } from './common/interface.js' import type { Database, Keyvalue } from 'welo' export interface KeyvalueDB extends Database { @@ -7,10 +8,9 @@ export interface KeyvalueDB extends Database { export const MEMORY_MAGIC = ':memory:' export interface ModuleMethod< - Context extends Record = Record, - Components extends Record = Record + Context extends Record = Record > { - (context: Context, components: Components): void + (components: Components, context: Context): void } // Optional type to get around linter void issues. From 8963ac6346b428b1ae8d85cf569beeb4f09473b9 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:12:49 +1300 Subject: [PATCH 17/62] Fix rpc commands. --- .../daemon/src/common/commands/addresses.ts | 10 +++ .../daemon/src/common/commands/connect.ts | 16 ++++ .../daemon/src/common/commands/connections.ts | 10 +++ .../daemon/src/common/commands/count-peers.ts | 36 ++++++++ .../src/common/commands/create-group.ts | 25 ++++++ .../daemon/src/common/commands/get-speeds.ts | 21 +++++ .../daemon/src/common/commands/get-status.ts | 28 +++++++ packages/daemon/src/common/commands/id.ts | 11 +++ .../daemon/src/common/commands/join-group.ts | 24 ++++++ .../daemon/src/common/commands/list-groups.ts | 16 ++++ .../src/common/commands/set-priority.ts | 23 ++++++ .../src/common/commands/sneakernet-receive.ts | 14 ++++ .../src/common/commands/sneakernet-send.ts | 14 ++++ packages/daemon/src/common/commands/sync.ts | 82 +++++++++++++++++++ packages/daemon/src/common/index.ts | 3 +- 15 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 packages/daemon/src/common/commands/addresses.ts create mode 100644 packages/daemon/src/common/commands/connect.ts create mode 100644 packages/daemon/src/common/commands/connections.ts create mode 100644 packages/daemon/src/common/commands/count-peers.ts create mode 100644 packages/daemon/src/common/commands/create-group.ts create mode 100644 packages/daemon/src/common/commands/get-speeds.ts create mode 100644 packages/daemon/src/common/commands/get-status.ts create mode 100644 packages/daemon/src/common/commands/id.ts create mode 100644 packages/daemon/src/common/commands/join-group.ts create mode 100644 packages/daemon/src/common/commands/list-groups.ts create mode 100644 packages/daemon/src/common/commands/set-priority.ts create mode 100644 packages/daemon/src/common/commands/sneakernet-receive.ts create mode 100644 packages/daemon/src/common/commands/sneakernet-send.ts create mode 100644 packages/daemon/src/common/commands/sync.ts diff --git a/packages/daemon/src/common/commands/addresses.ts b/packages/daemon/src/common/commands/addresses.ts new file mode 100644 index 00000000..aeb3c5fe --- /dev/null +++ b/packages/daemon/src/common/commands/addresses.ts @@ -0,0 +1,10 @@ +import { Addresses } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, libp2p }) => { + net.rpc.addMethod(Addresses.name, async (): Promise => { + return libp2p.getMultiaddrs().map(a => a.toString()) + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/connect.ts b/packages/daemon/src/common/commands/connect.ts new file mode 100644 index 00000000..7642083f --- /dev/null +++ b/packages/daemon/src/common/commands/connect.ts @@ -0,0 +1,16 @@ +import { multiaddr } from '@multiformats/multiaddr' +import { Connect } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, libp2p }) => { + net.rpc.addMethod(Connect.name, async (raw: unknown): Promise => { + const params = Connect.Params.parse(raw) + const address = multiaddr(params.address) + + await libp2p.dial(address) + + return null + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/connections.ts b/packages/daemon/src/common/commands/connections.ts new file mode 100644 index 00000000..381bade5 --- /dev/null +++ b/packages/daemon/src/common/commands/connections.ts @@ -0,0 +1,10 @@ +import { Connections } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, libp2p }) => { + net.rpc.addMethod(Connections.name, async (): Promise => { + return libp2p.getConnections().map(c => c.remoteAddr.toString()) + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/count-peers.ts b/packages/daemon/src/common/commands/count-peers.ts new file mode 100644 index 00000000..f59f9fd0 --- /dev/null +++ b/packages/daemon/src/common/commands/count-peers.ts @@ -0,0 +1,36 @@ +import { CountPeers } from '@organicdesign/db-rpc-interfaces' +import { CID } from 'multiformats/cid' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, libp2p }) => { + net.rpc.addMethod(CountPeers.name, async (raw: unknown): Promise => { + const countPeers = async (cid: CID, options?: { timeout: number }): Promise => { + let count = 0 + + const itr = libp2p.contentRouting.findProviders(cid, { + signal: AbortSignal.timeout(options?.timeout ?? 3000) + }) + + try { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _ of itr) { + count++ + } + } catch (error) { + // Do nothing + } + + return count + } + + const params = CountPeers.Params.parse(raw) + const cids = params.cids.map(cid => CID.parse(cid)) + + return Promise.all(cids.map(async cid => ({ + cid: cid.toString(), + peers: await countPeers(cid) + }))) + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/create-group.ts b/packages/daemon/src/common/commands/create-group.ts new file mode 100644 index 00000000..d5b8be46 --- /dev/null +++ b/packages/daemon/src/common/commands/create-group.ts @@ -0,0 +1,25 @@ +import { CreateGroup } from '@organicdesign/db-rpc-interfaces' +import { fromString as uint8ArrayFromString } from 'uint8arrays' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, welo, groups }) => { + net.rpc.addMethod(CreateGroup.name, async (raw: unknown): Promise => { + const params = CreateGroup.Params.parse(raw) + const peerValues = params.peers.map(p => uint8ArrayFromString(p, 'base58btc')) + + const manifest = await welo.determine({ + name: params.name, + meta: { type: 'group' }, + access: { + protocol: '/hldb/access/static', + config: { write: [welo.identity.id, ...peerValues] } + } + }) + + await groups.add(manifest) + + return manifest.address.cid.toString() + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/get-speeds.ts b/packages/daemon/src/common/commands/get-speeds.ts new file mode 100644 index 00000000..8cd35ae1 --- /dev/null +++ b/packages/daemon/src/common/commands/get-speeds.ts @@ -0,0 +1,21 @@ +import { GetSpeeds } from '@organicdesign/db-rpc-interfaces' +import { CID } from 'multiformats/cid' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, pinManager }) => { + net.rpc.addMethod(GetSpeeds.name, async (raw: unknown): Promise => { + const params = GetSpeeds.Params.parse(raw) + + return Promise.all(params.cids.map(async str => { + const cid = CID.parse(str) + const speed = await pinManager.getSpeed(cid, params.range) + + return { + cid: str, + speed + } + })) + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/get-status.ts b/packages/daemon/src/common/commands/get-status.ts new file mode 100644 index 00000000..ffbc5178 --- /dev/null +++ b/packages/daemon/src/common/commands/get-status.ts @@ -0,0 +1,28 @@ +import { GetStatus } from '@organicdesign/db-rpc-interfaces' +import { CID } from 'multiformats/cid' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ pinManager, net }) => { + net.rpc.addMethod(GetStatus.name, async (raw: unknown): Promise => { + const params = GetStatus.Params.parse(raw) + + return Promise.all(params.cids.map(async str => { + const cid = CID.parse(str) + + const [state, blocks, size] = await Promise.all([ + pinManager.getState(cid), + pinManager.getBlockCount(cid), + pinManager.getSize(cid) + ]) + + return { + cid: str, + state, + blocks, + size + } + })) + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/id.ts b/packages/daemon/src/common/commands/id.ts new file mode 100644 index 00000000..5e497ece --- /dev/null +++ b/packages/daemon/src/common/commands/id.ts @@ -0,0 +1,11 @@ +import { ID } from '@organicdesign/db-rpc-interfaces' +import { toString as uint8ArrayToString } from 'uint8arrays' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, welo }) => { + net.rpc.addMethod(ID.name, async (): Promise => { + return uint8ArrayToString(welo.identity.id, 'base58btc') + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/join-group.ts b/packages/daemon/src/common/commands/join-group.ts new file mode 100644 index 00000000..e0233761 --- /dev/null +++ b/packages/daemon/src/common/commands/join-group.ts @@ -0,0 +1,24 @@ +import { JoinGroup } from '@organicdesign/db-rpc-interfaces' +import { Address } from 'welo' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, welo, groups }) => { + net.rpc.addMethod(JoinGroup.name, async (raw: unknown): Promise => { + const params = JoinGroup.Params.parse(raw) + const manifest = await welo.fetch(Address.fromString(`/hldb/${params.group}`)) + + try { + await groups.add(manifest) + } catch (error) { + if ((error as Error).message.includes('is already open')) { + throw new Error('group has already been joined') + } + + throw error + } + + return null + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/list-groups.ts b/packages/daemon/src/common/commands/list-groups.ts new file mode 100644 index 00000000..e28ed8b6 --- /dev/null +++ b/packages/daemon/src/common/commands/list-groups.ts @@ -0,0 +1,16 @@ +import { ListGroups } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, groups }) => { + net.rpc.addMethod(ListGroups.name, async (): Promise => { + const promises: Array<{ group: string, name: string }> = [] + + for (const { key: cid, value: database } of groups.all()) { + promises.push({ group: cid, name: database.manifest.name }) + } + + return Promise.all(promises) + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/set-priority.ts b/packages/daemon/src/common/commands/set-priority.ts new file mode 100644 index 00000000..9331a17c --- /dev/null +++ b/packages/daemon/src/common/commands/set-priority.ts @@ -0,0 +1,23 @@ +import Path from 'path' +import { SetPriority } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, pinManager }) => { + net.rpc.addMethod(SetPriority.name, async (raw: unknown): Promise => { + const params = SetPriority.Params.parse(raw) + const key = Path.join('/', params.group, params.path) + const pinInfo = await pinManager.get(key) + + if (pinInfo == null) { + throw new Error('no such pin') + } + + pinInfo.priority = params.priority + + await pinManager.put(key, pinInfo) + + return null + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/sneakernet-receive.ts b/packages/daemon/src/common/commands/sneakernet-receive.ts new file mode 100644 index 00000000..2d909b34 --- /dev/null +++ b/packages/daemon/src/common/commands/sneakernet-receive.ts @@ -0,0 +1,14 @@ +import { SneakernetReveive } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ sneakernet, net }) => { + net.rpc.addMethod(SneakernetReveive.name, async (raw: unknown): Promise => { + const params = SneakernetReveive.Params.parse(raw) + + await sneakernet.import(params.path) + + return null + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/sneakernet-send.ts b/packages/daemon/src/common/commands/sneakernet-send.ts new file mode 100644 index 00000000..b6b18c31 --- /dev/null +++ b/packages/daemon/src/common/commands/sneakernet-send.ts @@ -0,0 +1,14 @@ +import { SneakernetSend } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ sneakernet, net }) => { + net.rpc.addMethod(SneakernetSend.name, async (raw: unknown): Promise => { + const params = SneakernetSend.Params.parse(raw) + + await sneakernet.export(params.path, params.peers) + + return null + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/sync.ts b/packages/daemon/src/common/commands/sync.ts new file mode 100644 index 00000000..e6d6392e --- /dev/null +++ b/packages/daemon/src/common/commands/sync.ts @@ -0,0 +1,82 @@ +import { Sync } from '@organicdesign/db-rpc-interfaces' +import { HeadsExchange } from 'welo/utils/heads-exchange' +import { cidstring } from 'welo/utils/index' +import { getHeads, addHeads } from 'welo/utils/replicator' +import type { ModuleMethod } from '@/interface.js' +import type { Peer, Libp2p } from '@libp2p/interface' +import type { Database } from 'welo' + +const sync = async (libp2p: Libp2p, peer: Peer, database: Database, options: Partial<{ reverseSync: boolean, collisionRate: number, validate: boolean, rounds: number }> = {}): Promise => { + if (!await libp2p.peerStore.has(peer.id)) { + await libp2p.peerStore.save(peer.id, peer) + } + + // We need to dial so that libp2p can update multiaddrs. + await libp2p.dial(peer.id) + + const protocol = `/hldb/replicator/he/1.0.0/${cidstring(database.manifest.address.cid)}` + const stream = await libp2p.dialProtocol(peer.id, protocol) + const heads = await getHeads(database.replica) + const he = new HeadsExchange({ + stream, + heads, + remotePeerId: peer.id, + collisionRate: options.collisionRate ?? 0.1, + localPeerId: libp2p.peerId + }) + + // eslint-disable-next-line no-console + const pipePromise = he.pipe().catch(console.error) + + if (!(options.reverseSync ?? true)) { + await pipePromise + he.close() + await stream.close() + return + } + + try { + for (let i = 0; i < (options.rounds ?? 5); i++) { + const seed = i === 0 ? undefined : Math.floor(Math.random() * Number.MAX_SAFE_INTEGER) + const newHeads = await he.getHeads(seed) + + await addHeads(newHeads, database.replica, database.components) + + if (options.validate ?? true) { + const matches = await he.verify() + + if (matches) { + break + } + } + } + } catch (error) { + // Ignore errors. + } + + he.close() + await stream.close() +} + +const command: ModuleMethod = ({ libp2p, welo, net }) => { + net.rpc.addMethod(Sync.name, async (): Promise => { + const peers = libp2p.getPeers() + const databases = welo.opened.values() + + const promises: Array> = [] + + for (const peerId of peers) { + const peer = await libp2p.peerStore.get(peerId) + + for (const database of databases) { + promises.push(sync(libp2p, peer, database)) + } + } + + await Promise.allSettled(promises) + + return null + }) +} + +export default command diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index fde5ffa4..5619beb6 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -155,7 +155,8 @@ export default async (): Promise => { getConfig, controller, groups, - pinManager + pinManager, + welo } return components From e2dd5a10220a4cfe22aa8e6ef812a11f4ad01aa4 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:16:48 +1300 Subject: [PATCH 18/62] Update index. --- packages/daemon/src/index.ts | 52 ++++++------------------------------ 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/packages/daemon/src/index.ts b/packages/daemon/src/index.ts index f0216dc7..f8dd344a 100644 --- a/packages/daemon/src/index.ts +++ b/packages/daemon/src/index.ts @@ -1,60 +1,24 @@ +import setupCommon from './common/index.js' import { createLogger } from './logger.js' -import setupArgv from '@/modules/argv/index.js' -import setupBase from '@/modules/base/index.js' -import setupConfig from '@/modules/config/index.js' -import setupDownloader from '@/modules/downloader/index.js' import setupFilesystem from '@/modules/filesystem/index.js' -import setupGroups from '@/modules/groups/index.js' -import setupNetwork from '@/modules/network/index.js' import setupRevisions from '@/modules/revisions/index.js' -import setupRPC from '@/modules/rpc/index.js' import setupScheduler from '@/modules/scheduler/index.js' -import setupSigint from '@/modules/sigint/index.js' -import setupSneakernet from '@/modules/sneakernet/index.js' -import setupTick from '@/modules/tick/index.js' const logger = createLogger('system') logger.info('starting...') // Setup all the modules -const argv = await setupArgv() +const components = await setupCommon() -const sigint = await setupSigint() -const config = await setupConfig({ argv }) -const rpc = await setupRPC({ argv, sigint }) -const tick = await setupTick({ config, sigint }) -const base = await setupBase({ argv, config }) -const network = await setupNetwork({ sigint, config, base, rpc }) -const groups = await setupGroups({ sigint, base, network, rpc }) -const downloader = await setupDownloader({ config, base, network, rpc, tick, sigint }) - -await setupSneakernet({ base, groups, rpc, network }) -await setupScheduler({ base, groups, rpc }) - -const filesystem = await setupFilesystem({ - config, - base, - network, - groups, - downloader, - tick, - rpc -}) - -await setupRevisions({ - config, - base, - network, - groups, - filesystem, - downloader, - tick, - rpc -}) +await Promise.all([ + setupScheduler(components), + setupFilesystem(components), + setupRevisions(components) +]) process.on('SIGINT', () => { - sigint.interupt().catch(() => {}) + components.controller.abort() }) logger.info('started') From e5b32e5bedb224902aed9cc28211b18e899846e9 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:19:58 +1300 Subject: [PATCH 19/62] Update module interface. --- packages/daemon/src/interface.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/daemon/src/interface.ts b/packages/daemon/src/interface.ts index 1b86b409..bdcea6af 100644 --- a/packages/daemon/src/interface.ts +++ b/packages/daemon/src/interface.ts @@ -17,11 +17,9 @@ export interface ModuleMethod< export type Optional = T extends undefined ? ReturnType<() => void> : T export interface Module< - Provides extends Record = Record, - Requires extends Record | undefined = undefined, - Init extends Record | undefined = undefined, + Context extends Record = Record > { - (components: Optional, init: Optional): Promise + (components: Components): Promise } export interface Pair { From b4f5d947345306e7486eecbd569876d9809af9fc Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:20:23 +1300 Subject: [PATCH 20/62] Fix scheduler module. --- .../scheduler/commands/get-schedule.ts | 6 +++--- .../scheduler/commands/put-schedule.ts | 6 +++--- .../daemon/src/modules/scheduler/index.ts | 19 +++++-------------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/packages/daemon/src/modules/scheduler/commands/get-schedule.ts b/packages/daemon/src/modules/scheduler/commands/get-schedule.ts index c368ad0a..c78889bd 100644 --- a/packages/daemon/src/modules/scheduler/commands/get-schedule.ts +++ b/packages/daemon/src/modules/scheduler/commands/get-schedule.ts @@ -1,11 +1,11 @@ import { GetSchedule } from '@organicdesign/db-rpc-interfaces' import all from 'it-all' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(GetSchedule.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, context) => { + net.rpc.addMethod(GetSchedule.name, async (raw: unknown): Promise => { const params = GetSchedule.Params.parse(raw) const group = CID.parse(params.group) const schedule = context.getSchedule(group) diff --git a/packages/daemon/src/modules/scheduler/commands/put-schedule.ts b/packages/daemon/src/modules/scheduler/commands/put-schedule.ts index d1884a42..1a0e6266 100644 --- a/packages/daemon/src/modules/scheduler/commands/put-schedule.ts +++ b/packages/daemon/src/modules/scheduler/commands/put-schedule.ts @@ -1,10 +1,10 @@ import { PutSchedule } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(PutSchedule.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, context) => { + net.rpc.addMethod(PutSchedule.name, async (raw: unknown): Promise => { const params = PutSchedule.Params.parse(raw) const group = CID.parse(params.group) const schedule = context.getSchedule(group) diff --git a/packages/daemon/src/modules/scheduler/index.ts b/packages/daemon/src/modules/scheduler/index.ts index 6060b836..a3ed5768 100644 --- a/packages/daemon/src/modules/scheduler/index.ts +++ b/packages/daemon/src/modules/scheduler/index.ts @@ -4,9 +4,6 @@ import setupGetSchedule from './commands/get-schedule.js' import setupPutSchedule from './commands/put-schedule.js' import { Schedule } from './schedule.js' import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as Groups } from '@/modules/groups/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' import type { CID } from 'multiformats/cid' import { createLogger } from '@/logger.js' @@ -19,31 +16,25 @@ export const Config = z.object({ // eslint-disable-next-line @typescript-eslint/no-redeclare export type Config = z.output -export interface Requires extends Record { - groups: Groups - base: Base - rpc: RPC -} - -export interface Provides extends Record { +export interface Context extends Record { getSchedule(group: CID): Schedule | null } -const module: Module = async (components) => { +const module: Module = async (components) => { const getSchedule = (group: CID): Schedule | null => { - const database = components.groups.groups.get(group) + const database = components.groups.get(group) if (database == null) { return null } - return new Schedule(database, components.groups.welo.identity.id) + return new Schedule(database, components.welo.identity.id) } const context = { getSchedule } for (const setupCommand of [setupGetSchedule, setupPutSchedule]) { - setupCommand(context, components) + setupCommand(components, context) } return context From 2ebf4939c9b7b710cfb3de068b6da606d5b40231 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:25:35 +1300 Subject: [PATCH 21/62] Fix most of the revisions module. --- .../revisions/commands/export-revision.ts | 8 +++--- .../revisions/commands/list-revisions.ts | 6 ++--- .../daemon/src/modules/revisions/index.ts | 25 +++---------------- .../daemon/src/modules/revisions/setup.ts | 9 ++++--- .../src/modules/revisions/sync-revisions.ts | 13 +++++----- 5 files changed, 22 insertions(+), 39 deletions(-) diff --git a/packages/daemon/src/modules/revisions/commands/export-revision.ts b/packages/daemon/src/modules/revisions/commands/export-revision.ts index 1eafc25f..d2567d79 100644 --- a/packages/daemon/src/modules/revisions/commands/export-revision.ts +++ b/packages/daemon/src/modules/revisions/commands/export-revision.ts @@ -3,11 +3,11 @@ import { ExportRevision } from '@organicdesign/db-rpc-interfaces' import { exporter } from '@organicdesign/db-utils' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, base, filesystem }) => { - rpc.addMethod(ExportRevision.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, blockstore }, context) => { + net.rpc.addMethod(ExportRevision.name, async (raw: unknown): Promise => { const params = ExportRevision.Params.parse(raw) const group = CID.parse(params.group) const revisions = context.getRevisions(group) @@ -29,7 +29,7 @@ const command: ModuleMethod = (context, { rpc, base, filesys } await exporter( - base.blockstore, + blockstore, outFile, revision.cid ) diff --git a/packages/daemon/src/modules/revisions/commands/list-revisions.ts b/packages/daemon/src/modules/revisions/commands/list-revisions.ts index a8a6f7f2..117efe53 100644 --- a/packages/daemon/src/modules/revisions/commands/list-revisions.ts +++ b/packages/daemon/src/modules/revisions/commands/list-revisions.ts @@ -1,11 +1,11 @@ import { ListRevisions } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' import { toString as uint8arrayToString } from 'uint8arrays/to-string' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(ListRevisions.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, context) => { + net.rpc.addMethod(ListRevisions.name, async (raw: unknown): Promise => { const params = ListRevisions.Params.parse(raw) const rs: ListRevisions.Return = [] const revisions = context.getRevisions(CID.parse(params.group)) diff --git a/packages/daemon/src/modules/revisions/index.ts b/packages/daemon/src/modules/revisions/index.ts index 534089a7..e9ef069a 100644 --- a/packages/daemon/src/modules/revisions/index.ts +++ b/packages/daemon/src/modules/revisions/index.ts @@ -7,14 +7,6 @@ import setup from './setup.js' import syncRevisions from './sync-revisions.js' import type { Revisions } from './revisions.js' import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as Downloader } from '@/modules/downloader/index.js' -import type { Provides as FileSystem } from '@/modules/filesystem/index.js' -import type { Provides as Groups } from '@/modules/groups/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Tick } from '@/modules/tick/index.js' import type { CID } from 'multiformats/cid' import { createLogger } from '@/logger.js' @@ -27,29 +19,18 @@ export const Config = z.object({ // eslint-disable-next-line @typescript-eslint/no-redeclare export type Config = z.output -export interface Requires extends Record { - base: Base - network: Network - groups: Groups - filesystem: FileSystem - rpc: RPC - config: ConfigModule - downloader: Downloader - tick: Tick -} - -export interface Provides extends Record { +export interface Context extends Record { getRevisions (group: CID): Revisions | null } -const module: Module = async (components) => { +const module: Module = async (components) => { const context = await setup(components) for (const setupCommand of [listRevisions, exportRevision, readRevision]) { setupCommand(context, components) } - components.tick.register(async () => syncRevisions(components)) + components.tick.add(async () => syncRevisions(components)) return context } diff --git a/packages/daemon/src/modules/revisions/setup.ts b/packages/daemon/src/modules/revisions/setup.ts index 7d91f99a..a75f3b7d 100644 --- a/packages/daemon/src/modules/revisions/setup.ts +++ b/packages/daemon/src/modules/revisions/setup.ts @@ -2,18 +2,19 @@ import all from 'it-all' import { Revisions } from './revisions.js' import selectRevisions from './select-revisions.js' import { pathToKey } from './utils.js' -import { type Provides, type Requires, logger } from './index.js' +import { type Context, logger } from './index.js' +import type { Components } from '@/common/interface.js' import type { CID } from 'multiformats/cid' -export default async ({ filesystem, groups }: Requires): Promise => { +export default async ({ groups, welo }: Components): Promise => { const getRevisions = (group: CID): Revisions | null => { - const database = groups.groups.get(group) + const database = groups.get(group) if (database == null) { return null } - return new Revisions(database, groups.welo.identity.id) + return new Revisions(database, welo.identity.id) } filesystem.events.addEventListener('file:added', ({ group, path, entry }) => { diff --git a/packages/daemon/src/modules/revisions/sync-revisions.ts b/packages/daemon/src/modules/revisions/sync-revisions.ts index 25e098c9..50db367f 100644 --- a/packages/daemon/src/modules/revisions/sync-revisions.ts +++ b/packages/daemon/src/modules/revisions/sync-revisions.ts @@ -2,11 +2,12 @@ import Path from 'path' import * as dagCbor from '@ipld/dag-cbor' import { VERSION_KEY, EncodedEntry } from './interface.js' import { decodeEntry } from './utils.js' -import { type Requires, logger } from './index.js' +import { logger } from './index.js' +import type { Components } from '@/common/interface.js' -export default async ({ groups, downloader }: Requires): Promise => { - for (const { value: database } of groups.groups.all()) { - const tracker = groups.getTracker(database) +export default async ({ groups, getTracker, pinManager }: Components): Promise => { + for (const { value: database } of groups.all()) { + const tracker = getTracker(database) const group = database.manifest.address.cid for await (const { key, value } of tracker.process(`/${VERSION_KEY}`)) { @@ -14,7 +15,7 @@ export default async ({ groups, downloader }: Requires): Promise => { const fullKey = Path.join('/', group.toString(), key) if (data == null) { - await downloader.pinManager.remove(fullKey) + await pinManager.remove(fullKey) continue } @@ -22,7 +23,7 @@ export default async ({ groups, downloader }: Requires): Promise => { logger.info('[entry] syncing update:', fullKey) - await downloader.pinManager.put(fullKey, { cid: entry.cid, priority: entry.priority }) + await pinManager.put(fullKey, { cid: entry.cid, priority: entry.priority }) } } } From 5c6dda31210bf5f90a4056cf9fbae4a43083e798 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:32:27 +1300 Subject: [PATCH 22/62] Add helia pin manager to components. --- packages/daemon/src/common/index.ts | 3 ++- packages/daemon/src/common/interface.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 5619beb6..54bd5566 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -156,7 +156,8 @@ export default async (): Promise => { controller, groups, pinManager, - welo + welo, + heliaPinManager } return components diff --git a/packages/daemon/src/common/interface.ts b/packages/daemon/src/common/interface.ts index d1f1315c..37707890 100644 --- a/packages/daemon/src/common/interface.ts +++ b/packages/daemon/src/common/interface.ts @@ -8,6 +8,7 @@ import type { Tick } from './tick.js' import type { KeyvalueDB } from '@/interface.js' import type { Helia } from '@helia/interface' import type { Libp2p } from '@libp2p/interface' +import type HeliaPinManager from '@organicdesign/db-helia-pin-manager' import type { NetServer } from '@organicdesign/net-rpc' import type { Blockstore } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' @@ -44,4 +45,5 @@ export interface Components { downloader: Downloader groups: Groups pinManager: PinManager + heliaPinManager: HeliaPinManager } From dede374d90075821d5799dc83ef55e387bc69195 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:34:18 +1300 Subject: [PATCH 23/62] Fix filesystem module. --- .../src/modules/filesystem/commands/delete.ts | 8 +++--- .../src/modules/filesystem/commands/export.ts | 8 +++--- .../src/modules/filesystem/commands/import.ts | 10 ++++---- .../src/modules/filesystem/commands/list.ts | 8 +++--- .../src/modules/filesystem/commands/read.ts | 8 +++--- .../src/modules/filesystem/commands/write.ts | 8 +++--- .../daemon/src/modules/filesystem/index.ts | 25 +++---------------- .../daemon/src/modules/filesystem/setup.ts | 17 +++++++------ .../src/modules/filesystem/sync-groups.ts | 13 +++++----- .../modules/filesystem/upload-operations.ts | 9 ++++--- 10 files changed, 50 insertions(+), 64 deletions(-) diff --git a/packages/daemon/src/modules/filesystem/commands/delete.ts b/packages/daemon/src/modules/filesystem/commands/delete.ts index 7397133a..2c835adb 100644 --- a/packages/daemon/src/modules/filesystem/commands/delete.ts +++ b/packages/daemon/src/modules/filesystem/commands/delete.ts @@ -1,12 +1,12 @@ import { Delete } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Delete.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, { uploads }) => { + net.rpc.addMethod(Delete.name, async (raw: unknown): Promise => { const params = Delete.Params.parse(raw) - const pairs = await context.uploads.add('delete', [CID.parse(params.group).bytes, params.path]) + const pairs = await uploads.add('delete', [CID.parse(params.group).bytes, params.path]) return pairs.map(p => ({ path: p.key, cid: p.value.cid.toString() })) }) diff --git a/packages/daemon/src/modules/filesystem/commands/export.ts b/packages/daemon/src/modules/filesystem/commands/export.ts index 9252480f..73b0e994 100644 --- a/packages/daemon/src/modules/filesystem/commands/export.ts +++ b/packages/daemon/src/modules/filesystem/commands/export.ts @@ -2,11 +2,11 @@ import Path from 'path' import { Export } from '@organicdesign/db-rpc-interfaces' import { exporter } from '@organicdesign/db-utils' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, base }) => { - rpc.addMethod(Export.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, blockstore }, context) => { + net.rpc.addMethod(Export.name, async (raw: unknown): Promise => { const params = Export.Params.parse(raw) const fs = context.getFileSystem(CID.parse(params.group)) @@ -16,7 +16,7 @@ const command: ModuleMethod = (context, { rpc, base }) => { for await (const pair of fs.getDir(params.path)) { await exporter( - base.blockstore, + blockstore, Path.join(params.outPath, pair.key.toString().replace(params.path, '')), pair.value.cid ) diff --git a/packages/daemon/src/modules/filesystem/commands/import.ts b/packages/daemon/src/modules/filesystem/commands/import.ts index ca29240f..1900b08e 100644 --- a/packages/daemon/src/modules/filesystem/commands/import.ts +++ b/packages/daemon/src/modules/filesystem/commands/import.ts @@ -3,11 +3,11 @@ import { Import } from '@organicdesign/db-rpc-interfaces' import { selectChunker, importer, type ImporterConfig } from '@organicdesign/db-utils' import { BlackHoleBlockstore } from 'blockstore-core/black-hole' import { CID } from 'multiformats/cid' -import { type Provides, type Requires, logger } from '../index.js' +import { type Context, logger } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, network, base }) => { - rpc.addMethod(Import.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, blockstore, heliaPinManager }, context) => { + net.rpc.addMethod(Import.name, async (raw: unknown): Promise => { const params = Import.Params.parse(raw) const encrypt = Boolean(params.encrypt) @@ -24,12 +24,12 @@ const command: ModuleMethod = (context, { rpc, network, base logger.info('[add] importing %s', params.inPath) } - const store = params.onlyHash ? new BlackHoleBlockstore() : base.blockstore + const store = params.onlyHash ? new BlackHoleBlockstore() : blockstore const cids: Import.Return = [] for await (const r of importer(store, params.inPath, config)) { - await network.pinManager.pinLocal(r.cid) + await heliaPinManager.pinLocal(r.cid) logger.info('[add] imported %s', params.inPath) diff --git a/packages/daemon/src/modules/filesystem/commands/list.ts b/packages/daemon/src/modules/filesystem/commands/list.ts index cbbe627d..499f7fe5 100644 --- a/packages/daemon/src/modules/filesystem/commands/list.ts +++ b/packages/daemon/src/modules/filesystem/commands/list.ts @@ -1,15 +1,15 @@ import { List } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' import { toString as uint8arrayToString } from 'uint8arrays' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, groups }) => { - rpc.addMethod(List.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, groups }, context) => { + net.rpc.addMethod(List.name, async (raw: unknown): Promise => { const params = List.Params.parse(raw) const list: List.Return = [] - for (const { key: group } of groups.groups.all()) { + for (const { key: group } of groups.all()) { if (params.group != null && group !== params.group) { continue } diff --git a/packages/daemon/src/modules/filesystem/commands/read.ts b/packages/daemon/src/modules/filesystem/commands/read.ts index bc514f02..89d20a79 100644 --- a/packages/daemon/src/modules/filesystem/commands/read.ts +++ b/packages/daemon/src/modules/filesystem/commands/read.ts @@ -4,11 +4,11 @@ import { CID } from 'multiformats/cid' import { collect } from 'streaming-iterables' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, network }) => { - rpc.addMethod(Read.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, helia }, context) => { + net.rpc.addMethod(Read.name, async (raw: unknown): Promise => { const params = Read.Params.parse(raw) const fs = context.getFileSystem(CID.parse(params.group)) @@ -22,7 +22,7 @@ const command: ModuleMethod = (context, { rpc, network }) => throw new Error(`no such item: ${params.path}`) } - const ufs = unixfs(network.helia) + const ufs = unixfs(helia) return uint8ArrayToString(uint8ArrayConcat(await collect(ufs.cat(entry.cid, { offset: params.position, length: params.length })))) }) diff --git a/packages/daemon/src/modules/filesystem/commands/write.ts b/packages/daemon/src/modules/filesystem/commands/write.ts index afcff609..b7452d52 100644 --- a/packages/daemon/src/modules/filesystem/commands/write.ts +++ b/packages/daemon/src/modules/filesystem/commands/write.ts @@ -5,12 +5,12 @@ import { CID } from 'multiformats/cid' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { FileSystemEvent } from '../events.js' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { Entry } from '../interface.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, network }) => { - rpc.addMethod(Write.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, helia }, context) => { + net.rpc.addMethod(Write.name, async (raw: unknown): Promise => { const params = Write.Params.parse(raw) const group = CID.parse(params.group) const fs = context.getFileSystem(CID.parse(params.group)) @@ -20,7 +20,7 @@ const command: ModuleMethod = (context, { rpc, network }) => } const entry: Partial = await fs.get(params.path) ?? {} - const ufs = unixfs(network.helia) + const ufs = unixfs(helia) const existingData = (entry.cid != null) ? uint8ArrayConcat(await all(ufs.cat(entry.cid))) : new Uint8Array() diff --git a/packages/daemon/src/modules/filesystem/index.ts b/packages/daemon/src/modules/filesystem/index.ts index 49ca8ab1..b47e4796 100644 --- a/packages/daemon/src/modules/filesystem/index.ts +++ b/packages/daemon/src/modules/filesystem/index.ts @@ -14,13 +14,6 @@ import type { Events } from './events.js' import type { LocalSettings } from './local-settings.js' import type createUploadManager from './upload-operations.js' import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as Downloader } from '@/modules/downloader/index.js' -import type { Provides as Groups } from '@/modules/groups/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Tick } from '@/modules/tick/index.js' import type { CID } from 'multiformats/cid' import { createLogger } from '@/logger.js' @@ -33,17 +26,7 @@ export const Config = z.object({ // eslint-disable-next-line @typescript-eslint/no-redeclare export type Config = z.output -export interface Requires extends Record { - base: Base - network: Network - groups: Groups - downloader: Downloader - tick: Tick - rpc: RPC - config: ConfigModule -} - -export interface Provides extends Record { +export interface Context extends Record { uploads: Awaited> localSettings: LocalSettings config: Config @@ -51,8 +34,8 @@ export interface Provides extends Record { events: Events } -const module: Module = async (components) => { - const config = components.config.get(Config) +const module: Module = async (components) => { + const config = components.getConfig(Config) const context = await setup(components, config) for (const setupCommand of [ @@ -67,7 +50,7 @@ const module: Module = async (components) => { setupCommand(context, components) } - components.tick.register(async () => syncGroups(components, context)) + components.tick.add(async () => syncGroups(components, context)) return context } diff --git a/packages/daemon/src/modules/filesystem/setup.ts b/packages/daemon/src/modules/filesystem/setup.ts index 2ea7750e..73023461 100644 --- a/packages/daemon/src/modules/filesystem/setup.ts +++ b/packages/daemon/src/modules/filesystem/setup.ts @@ -3,19 +3,21 @@ import { FileSystem } from './file-system.js' import { LocalSettings } from './local-settings.js' import createUploadManager from './upload-operations.js' import type { Events } from './events.js' -import type { Requires, Provides, Config } from './index.js' +import type { Context, Config } from './index.js' +import type { Components } from '@/common/interface.js' import type { CID } from 'multiformats/cid' import { extendDatastore } from '@/utils.js' -export default async (components: Requires, config: Config): Promise => { +export default async (components: Components, config: Config): Promise => { + const { groups, datastore, blockstore, welo } = components const events: Events = new EventTarget() const localSettings = new LocalSettings({ - datastore: extendDatastore(components.base.datastore, 'references') + datastore: extendDatastore(datastore, 'references') }) const getFileSystem = (group: CID): FileSystem | null => { - const database = components.groups.groups.get(group) + const database = groups.get(group) if (database == null) { return null @@ -23,9 +25,8 @@ export default async (components: Requires, config: Config): Promise = return new FileSystem({ database, - blockstore: - components.base.blockstore, - id: components.groups.welo.identity.id, + blockstore, + id: welo.identity.id, localSettings }) } @@ -33,7 +34,7 @@ export default async (components: Requires, config: Config): Promise = const uploads = await createUploadManager( { getFileSystem, events }, components, - extendDatastore(components.base.datastore, 'upload-operations') + extendDatastore(datastore, 'upload-operations') ) return { diff --git a/packages/daemon/src/modules/filesystem/sync-groups.ts b/packages/daemon/src/modules/filesystem/sync-groups.ts index 07ab4085..00304311 100644 --- a/packages/daemon/src/modules/filesystem/sync-groups.ts +++ b/packages/daemon/src/modules/filesystem/sync-groups.ts @@ -1,11 +1,12 @@ import Path from 'path' import { DATA_KEY } from './interface.js' import { keyToPath } from './utils.js' -import { type Requires, type Provides, logger } from './index.js' +import { type Context, logger } from './index.js' +import type { Components } from '@/common/interface.js' -export default async ({ groups, downloader }: Requires, context: Provides): Promise => { - for (const { value: database } of groups.groups.all()) { - const tracker = groups.getTracker(database) +export default async ({ getTracker, groups, pinManager }: Components, context: Context): Promise => { + for (const { value: database } of groups.all()) { + const tracker = getTracker(database) for await (const { key } of tracker.process(`/${DATA_KEY}`)) { const path = keyToPath(key) @@ -22,11 +23,11 @@ export default async ({ groups, downloader }: Requires, context: Provides): Prom logger.info('[entry] syncing update:', fullKey) if (entry == null) { - await downloader.pinManager.remove(fullKey) + await pinManager.remove(fullKey) continue } - await downloader.pinManager.put(fullKey, { cid: entry.cid, priority: entry.priority }) + await pinManager.put(fullKey, { cid: entry.cid, priority: entry.priority }) } } } diff --git a/packages/daemon/src/modules/filesystem/upload-operations.ts b/packages/daemon/src/modules/filesystem/upload-operations.ts index 255badf5..ef1a36ff 100644 --- a/packages/daemon/src/modules/filesystem/upload-operations.ts +++ b/packages/daemon/src/modules/filesystem/upload-operations.ts @@ -5,13 +5,14 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { take } from 'streaming-iterables' import { FileSystemEvent } from './events.js' -import type { Requires, Provides } from './index.js' +import type { Context } from './index.js' import type { Entry } from './interface.js' +import type { Components } from '@/common/interface.js' import type { Pair } from '@/interface.js' import type { Datastore } from 'interface-datastore' import { OperationManager } from '@/operation-manager.js' -export default async (context: Pick, { network, downloader }: Requires, datastore: Datastore): Promise, { pinManager, helia }: Components, datastore: Datastore): Promise delete(groupData: Uint8Array, path: string): Promise>> }>> => { @@ -25,7 +26,7 @@ export default async (context: Pick, { net } const fullEntry = await fs.put(path, { ...entry, cid }) - await downloader.pinManager.put(path, { cid, priority: entry.priority }) + await pinManager.put(path, { cid, priority: entry.priority }) context.events.dispatchEvent(new FileSystemEvent('file:added', group, path, fullEntry)) } @@ -41,7 +42,7 @@ export default async (context: Pick, { net } const parentPath = path.split('/').slice(0, -2).join('/') - const ufs = unixfs(network.helia) + const ufs = unixfs(helia) const cid = await ufs.addBytes(new Uint8Array([])) const pairs = await all(fs.getDir(path)) From 07a3dcf7e9467c43688427303fd9f4ece63b36e9 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:34:51 +1300 Subject: [PATCH 24/62] Nuke old modules. --- packages/daemon/src/modules/argv/index.ts | 41 ------ packages/daemon/src/modules/base/index.ts | 45 ------ packages/daemon/src/modules/config/index.ts | 23 --- .../modules/downloader/commands/get-speeds.ts | 22 --- .../modules/downloader/commands/get-status.ts | 29 ---- .../downloader/commands/set-priority.ts | 24 ---- .../src/modules/downloader/downloader.ts | 71 --------- .../daemon/src/modules/downloader/index.ts | 63 -------- .../src/modules/downloader/interface.ts | 15 -- .../src/modules/downloader/pin-manager.ts | 124 ---------------- .../daemon/src/modules/downloader/utils.ts | 38 ----- .../modules/groups/commands/create-group.ts | 26 ---- .../daemon/src/modules/groups/commands/id.ts | 12 -- .../src/modules/groups/commands/join-group.ts | 25 ---- .../modules/groups/commands/list-groups.ts | 17 --- .../src/modules/groups/commands/sync.ts | 83 ----------- .../src/modules/groups/entry-tracker.ts | 67 --------- packages/daemon/src/modules/groups/groups.ts | 84 ----------- packages/daemon/src/modules/groups/index.ts | 53 ------- packages/daemon/src/modules/groups/setup.ts | 28 ---- .../src/modules/network/commands/addresses.ts | 11 -- .../src/modules/network/commands/connect.ts | 17 --- .../modules/network/commands/connections.ts | 11 -- .../modules/network/commands/count-peers.ts | 37 ----- packages/daemon/src/modules/network/index.ts | 65 --------- packages/daemon/src/modules/network/libp2p.ts | 73 ---------- packages/daemon/src/modules/network/setup.ts | 57 -------- packages/daemon/src/modules/rpc/index.ts | 27 ---- packages/daemon/src/modules/sigint/index.ts | 43 ------ .../sneakernet/commands/sneakernet-receive.ts | 15 -- .../sneakernet/commands/sneakernet-send.ts | 15 -- .../daemon/src/modules/sneakernet/index.ts | 49 ------- .../src/modules/sneakernet/interface.ts | 22 --- .../src/modules/sneakernet/sneakernet.ts | 135 ------------------ packages/daemon/src/modules/tick/index.ts | 72 ---------- 35 files changed, 1539 deletions(-) delete mode 100644 packages/daemon/src/modules/argv/index.ts delete mode 100644 packages/daemon/src/modules/base/index.ts delete mode 100644 packages/daemon/src/modules/config/index.ts delete mode 100644 packages/daemon/src/modules/downloader/commands/get-speeds.ts delete mode 100644 packages/daemon/src/modules/downloader/commands/get-status.ts delete mode 100644 packages/daemon/src/modules/downloader/commands/set-priority.ts delete mode 100644 packages/daemon/src/modules/downloader/downloader.ts delete mode 100644 packages/daemon/src/modules/downloader/index.ts delete mode 100644 packages/daemon/src/modules/downloader/interface.ts delete mode 100644 packages/daemon/src/modules/downloader/pin-manager.ts delete mode 100644 packages/daemon/src/modules/downloader/utils.ts delete mode 100644 packages/daemon/src/modules/groups/commands/create-group.ts delete mode 100644 packages/daemon/src/modules/groups/commands/id.ts delete mode 100644 packages/daemon/src/modules/groups/commands/join-group.ts delete mode 100644 packages/daemon/src/modules/groups/commands/list-groups.ts delete mode 100644 packages/daemon/src/modules/groups/commands/sync.ts delete mode 100644 packages/daemon/src/modules/groups/entry-tracker.ts delete mode 100644 packages/daemon/src/modules/groups/groups.ts delete mode 100644 packages/daemon/src/modules/groups/index.ts delete mode 100644 packages/daemon/src/modules/groups/setup.ts delete mode 100644 packages/daemon/src/modules/network/commands/addresses.ts delete mode 100644 packages/daemon/src/modules/network/commands/connect.ts delete mode 100644 packages/daemon/src/modules/network/commands/connections.ts delete mode 100644 packages/daemon/src/modules/network/commands/count-peers.ts delete mode 100644 packages/daemon/src/modules/network/index.ts delete mode 100644 packages/daemon/src/modules/network/libp2p.ts delete mode 100644 packages/daemon/src/modules/network/setup.ts delete mode 100644 packages/daemon/src/modules/rpc/index.ts delete mode 100644 packages/daemon/src/modules/sigint/index.ts delete mode 100644 packages/daemon/src/modules/sneakernet/commands/sneakernet-receive.ts delete mode 100644 packages/daemon/src/modules/sneakernet/commands/sneakernet-send.ts delete mode 100644 packages/daemon/src/modules/sneakernet/index.ts delete mode 100644 packages/daemon/src/modules/sneakernet/interface.ts delete mode 100644 packages/daemon/src/modules/sneakernet/sneakernet.ts delete mode 100644 packages/daemon/src/modules/tick/index.ts diff --git a/packages/daemon/src/modules/argv/index.ts b/packages/daemon/src/modules/argv/index.ts deleted file mode 100644 index 0e420d8f..00000000 --- a/packages/daemon/src/modules/argv/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import Path from 'path' -import { hideBin } from 'yargs/helpers' -import yargs from 'yargs/yargs' -import type { Module } from '@/interface.js' -import { projectPath } from '@/utils.js' - -export interface Provides extends Record { - socket: string - key: string - config: string -} - -const module: Module = async () => { - const argv = await yargs(hideBin(process.argv)) - .option({ - socket: { - alias: 's', - type: 'string', - default: '/tmp/server.socket' - } - }) - .option({ - key: { - alias: 'k', - type: 'string', - default: Path.join(projectPath, 'config/key.json') - } - }) - .option({ - config: { - alias: 'c', - type: 'string', - default: Path.join(projectPath, 'config/config.json') - } - }) - .parse() - - return { socket: argv.socket, key: argv.key, config: argv.config } -} - -export default module diff --git a/packages/daemon/src/modules/base/index.ts b/packages/daemon/src/modules/base/index.ts deleted file mode 100644 index 80231ca9..00000000 --- a/packages/daemon/src/modules/base/index.ts +++ /dev/null @@ -1,45 +0,0 @@ -import Path from 'path' -import { createKeyManager, type KeyManager } from '@organicdesign/db-key-manager' -import { MemoryBlockstore } from 'blockstore-core' -import { FsBlockstore } from 'blockstore-fs' -import { MemoryDatastore } from 'datastore-core' -import { FsDatastore } from 'datastore-fs' -import { z } from 'zod' -import type { Module } from '@/interface.js' -import type { Provides as Argv } from '@/modules/argv/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Blockstore } from 'interface-blockstore' -import type { Datastore } from 'interface-datastore' -import { isMemory } from '@/utils.js' - -const Config = z.object({ - storage: z.string().default(':memory:') -}) - -export interface Requires extends Record { - argv: Argv - config: ConfigModule -} - -export interface Provides extends Record { - datastore: Datastore - blockstore: Blockstore - keyManager: KeyManager -} - -const module: Module = async ({ argv, config }) => { - const c = config.get(Config) - const keyManager = await createKeyManager(Path.resolve(argv.key)) - - const datastore = isMemory(c.storage) - ? new MemoryDatastore() - : new FsDatastore(Path.join(c.storage, 'datastore')) - - const blockstore = isMemory(c.storage) - ? new MemoryBlockstore() - : new FsBlockstore(Path.join(c.storage, 'blockstore')) - - return { keyManager, datastore, blockstore } -} - -export default module diff --git a/packages/daemon/src/modules/config/index.ts b/packages/daemon/src/modules/config/index.ts deleted file mode 100644 index 586d160f..00000000 --- a/packages/daemon/src/modules/config/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -import fs from 'fs/promises' -import { z } from 'zod' -import type { Module } from '@/interface.js' -import type { Provides as Argv } from '@/modules/argv/index.js' - -export interface Requires extends Record { - argv: Argv -} - -export interface Provides extends Record { - config: Record - get (shape: T): z.infer -} - -const module: Module = async ({ argv }) => { - const raw = await fs.readFile(argv.config, { encoding: 'utf8' }) - const config = z.record(z.unknown()).parse(JSON.parse(raw)) - const get = (shape: T): z.infer => shape.parse(config) - - return { config, get } -} - -export default module diff --git a/packages/daemon/src/modules/downloader/commands/get-speeds.ts b/packages/daemon/src/modules/downloader/commands/get-speeds.ts deleted file mode 100644 index a612f37c..00000000 --- a/packages/daemon/src/modules/downloader/commands/get-speeds.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { GetSpeeds } from '@organicdesign/db-rpc-interfaces' -import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(GetSpeeds.name, async (raw: unknown): Promise => { - const params = GetSpeeds.Params.parse(raw) - - return Promise.all(params.cids.map(async str => { - const cid = CID.parse(str) - const speed = await context.pinManager.getSpeed(cid, params.range) - - return { - cid: str, - speed - } - })) - }) -} - -export default command diff --git a/packages/daemon/src/modules/downloader/commands/get-status.ts b/packages/daemon/src/modules/downloader/commands/get-status.ts deleted file mode 100644 index a127b3b8..00000000 --- a/packages/daemon/src/modules/downloader/commands/get-status.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { GetStatus } from '@organicdesign/db-rpc-interfaces' -import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(GetStatus.name, async (raw: unknown): Promise => { - const params = GetStatus.Params.parse(raw) - - return Promise.all(params.cids.map(async str => { - const cid = CID.parse(str) - - const [state, blocks, size] = await Promise.all([ - context.pinManager.getState(cid), - context.pinManager.getBlockCount(cid), - context.pinManager.getSize(cid) - ]) - - return { - cid: str, - state, - blocks, - size - } - })) - }) -} - -export default command diff --git a/packages/daemon/src/modules/downloader/commands/set-priority.ts b/packages/daemon/src/modules/downloader/commands/set-priority.ts deleted file mode 100644 index 9a845fef..00000000 --- a/packages/daemon/src/modules/downloader/commands/set-priority.ts +++ /dev/null @@ -1,24 +0,0 @@ -import Path from 'path' -import { SetPriority } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(SetPriority.name, async (raw: unknown): Promise => { - const params = SetPriority.Params.parse(raw) - const key = Path.join('/', params.group, params.path) - const pinInfo = await context.pinManager.get(key) - - if (pinInfo == null) { - throw new Error('no such pin') - } - - pinInfo.priority = params.priority - - await context.pinManager.put(key, pinInfo) - - return null - }) -} - -export default command diff --git a/packages/daemon/src/modules/downloader/downloader.ts b/packages/daemon/src/modules/downloader/downloader.ts deleted file mode 100644 index 14cdf65d..00000000 --- a/packages/daemon/src/modules/downloader/downloader.ts +++ /dev/null @@ -1,71 +0,0 @@ -import parallel from 'it-parallel' -import { pipe } from 'it-pipe' -import { type CID } from 'multiformats/cid' -import { collect } from 'streaming-iterables' -import { linearWeightTranslation } from './utils.js' -import { type Provides, logger } from './index.js' - -export default async (context: Provides, options?: { signal: AbortSignal }): Promise => { - const batchDownload = async function * (itr: AsyncIterable<[CID, number]>): AsyncGenerator<() => Promise<{ cid: CID, block: Uint8Array }>, void, undefined> { - for await (const [cid, priority] of itr) { - if (options?.signal.aborted === true) { - return - } - - const weight = Math.floor(linearWeightTranslation(priority / 100) * context.config.slots) + 1 - const downloaders = await context.pinManager.download(cid, { limit: weight }) - - yield * downloaders - } - } - - const catcher = async function * (itr: AsyncIterable): AsyncIterable { - try { - yield * itr - } catch (error) { - logger.warn('downloader threw: ', error) - yield * catcher(itr) - } - } - - const loop = async function * (): AsyncGenerator { - for (;;) { - if (options?.signal.aborted === true) { - return - } - - await new Promise(resolve => setTimeout(resolve, 100)) - yield - } - } - - const getPins = async function * (loop: AsyncIterable): AsyncGenerator<[CID, number]> { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const _ of loop) { - for await (const { value } of context.pinManager.getActive()) { - if (options?.signal.aborted === true) { - return - } - - yield [value.cid, value.priority] - } - } - } - - for (;;) { - if (options?.signal.aborted === true) { - return - } - - await pipe( - loop, - getPins, - batchDownload, - i => parallel(i, { concurrency: context.config.slots, ordered: false }), - i => catcher(i), - async i => collect(i) - ) - - await new Promise(resolve => setTimeout(resolve, 100)) - } -} diff --git a/packages/daemon/src/modules/downloader/index.ts b/packages/daemon/src/modules/downloader/index.ts deleted file mode 100644 index 5ce1c2af..00000000 --- a/packages/daemon/src/modules/downloader/index.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { z } from 'zod' -import getSpeeds from './commands/get-speeds.js' -import getStatus from './commands/get-status.js' -import setPriority from './commands/set-priority.js' -import download from './downloader.js' -import { PinManager } from './pin-manager.js' -import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' -import { createLogger } from '@/logger.js' -import { extendDatastore } from '@/utils.js' - -export const logger = createLogger('downloader') - -export const Config = z.object({ - slots: z.number().int().min(1).max(100).default(20) -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type Config = z.output - -export interface Requires extends Record { - base: Base - network: Network - rpc: RPC - config: ConfigModule - sigint: Sigint -} - -export interface Provides extends Record { - pinManager: PinManager - config: Config -} - -const module: Module = async (components) => { - const c = components.config.get(Config) - - const pinManager = new PinManager({ - pinManager: components.network.pinManager, - datastore: extendDatastore(components.base.datastore, 'pin-references') - }) - - const context = { pinManager, config: c } - - for (const setupCommand of [setPriority, getStatus, getSpeeds]) { - setupCommand(context, components) - } - - const controller = new AbortController() - - download(context, { signal: controller.signal }).catch(error => { - throw error - }) - - components.sigint.onInterupt(() => { controller.abort() }) - - return context -} - -export default module diff --git a/packages/daemon/src/modules/downloader/interface.ts b/packages/daemon/src/modules/downloader/interface.ts deleted file mode 100644 index 27546291..00000000 --- a/packages/daemon/src/modules/downloader/interface.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { z } from 'zod' -import type { CID } from 'multiformats/cid' - -export const EncodedPinInfo = z.object({ - priority: z.number().int().min(1).max(100).optional().default(100), - cid: z.instanceof(Uint8Array) -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type EncodedPinInfo = z.output - -export interface PinInfo { - priority: number - cid: CID -} diff --git a/packages/daemon/src/modules/downloader/pin-manager.ts b/packages/daemon/src/modules/downloader/pin-manager.ts deleted file mode 100644 index e441570d..00000000 --- a/packages/daemon/src/modules/downloader/pin-manager.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { Key, type Datastore } from 'interface-datastore' -import all from 'it-all' -import { type CID } from 'multiformats/cid' -import { encodePinInfo, decodePinInfo } from './utils.js' -import { logger } from './index.js' -import type { PinInfo } from './interface.js' -import type { Pair } from '@/interface.js' -import type { BlockInfo } from '@organicdesign/db-helia-pin-manager' -import type HeliaPinManager from '@organicdesign/db-helia-pin-manager' - -export class PinManager { - private readonly datastore: Datastore - private readonly pinManager: HeliaPinManager - - constructor (components: { datastore: Datastore, pinManager: HeliaPinManager }) { - this.datastore = components.datastore - this.pinManager = components.pinManager - } - - async put (key: string, pinInfo: PinInfo): Promise { - const data = encodePinInfo(pinInfo) - - // Need to ensure that the references get updated. - await this.remove(key) - - await this.pinManager.pin(pinInfo.cid) - - await this.datastore.put(new Key(key), data) - } - - async has (key: string, cid?: CID): Promise { - const pinInfo = await this.get(key) - - if (pinInfo == null) { - return false - } - - if (cid == null) { - return true - } - - return pinInfo.cid.equals(cid) - } - - async * getActive (): AsyncGenerator> { - for (const pin of await this.pinManager.getActiveDownloads()) { - yield * this.getByPin(pin) - } - } - - async download (pin: CID, options?: { limit: number }): Promise Promise>> { - return this.pinManager.downloadSync(pin, options) - } - - async getState (cid: CID): Promise<'COMPLETED' | 'DOWNLOADING' | 'DESTROYED' | 'UPLOADING' | 'NOTFOUND'> { - return this.pinManager.getState(cid) - } - - async getSpeed (cid: CID, range?: number): Promise { - return this.pinManager.getSpeed(cid, range) - } - - async getSize (cid: CID): Promise { - return this.pinManager.getSize(cid) - } - - async getBlockCount (cid: CID): Promise { - return this.pinManager.getBlockCount(cid) - } - - async remove (key: string): Promise { - const pinInfo = await this.get(key) - - if (pinInfo == null) { - return - } - - const keys = await all(this.getByPin(pinInfo.cid)) - - // If we only have 1 reference be sure to unpin it. - if (keys.length <= 1) { - await this.pinManager.unpin(pinInfo.cid) - } - - logger.info(`[references] [-] ${key.toString()}`) - - await this.datastore.delete(new Key(key)) - } - - async get (key: string): Promise { - try { - const data = await this.datastore.get(new Key(key)) - - return decodePinInfo(data) - } catch (error) { - return null - } - } - - private async * getByPin (pin: CID): AsyncGenerator> { - const itr = this.datastore.query({ - filters: [({ value }) => { - const pinInfo = decodePinInfo(value) - - if (pinInfo == null) { - return false - } - - return pinInfo.cid.equals(pin) - }] - }) - - for await (const pair of itr) { - const pinInfo = decodePinInfo(pair.value) - - if (pinInfo == null) { - await this.datastore.delete(pair.key) - continue - } - - yield { key: pair.key.toString(), value: pinInfo } - } - } -} diff --git a/packages/daemon/src/modules/downloader/utils.ts b/packages/daemon/src/modules/downloader/utils.ts deleted file mode 100644 index e845b3eb..00000000 --- a/packages/daemon/src/modules/downloader/utils.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { CID } from 'multiformats/cid' -import { EncodedPinInfo, type PinInfo } from './interface.js' -import { decodeAny, encodeAny } from '@/utils.js' - -export const linearWeightTranslation = (p: number): number => { - return 1 - p -} - -export const logWeightTranslation = (p: number): number => { - return 1 - Math.log10((10 - 1) * p - 1) -} - -export const decodePinInfo = (data: Uint8Array): PinInfo | null => { - const obj = decodeAny(data) - - if (obj == null) { - return null - } - - const encodedPinInfo = EncodedPinInfo.parse(obj) - - return { - ...encodedPinInfo, - cid: CID.decode(encodedPinInfo.cid) - } -} - -export const encodePinInfo = (pinInfo: PinInfo): Uint8Array => { - const encodedPinInfo: EncodedPinInfo = { - ...pinInfo, - cid: pinInfo.cid.bytes - } - - // This will strip foreign keys. - const parsedEncodedPinInfo = EncodedPinInfo.parse(encodedPinInfo) - - return encodeAny(parsedEncodedPinInfo) -} diff --git a/packages/daemon/src/modules/groups/commands/create-group.ts b/packages/daemon/src/modules/groups/commands/create-group.ts deleted file mode 100644 index f8de7451..00000000 --- a/packages/daemon/src/modules/groups/commands/create-group.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { CreateGroup } from '@organicdesign/db-rpc-interfaces' -import { fromString as uint8ArrayFromString } from 'uint8arrays' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(CreateGroup.name, async (raw: unknown): Promise => { - const params = CreateGroup.Params.parse(raw) - const peerValues = params.peers.map(p => uint8ArrayFromString(p, 'base58btc')) - - const manifest = await context.welo.determine({ - name: params.name, - meta: { type: 'group' }, - access: { - protocol: '/hldb/access/static', - config: { write: [context.welo.identity.id, ...peerValues] } - } - }) - - await context.groups.add(manifest) - - return manifest.address.cid.toString() - }) -} - -export default command diff --git a/packages/daemon/src/modules/groups/commands/id.ts b/packages/daemon/src/modules/groups/commands/id.ts deleted file mode 100644 index ea21d97b..00000000 --- a/packages/daemon/src/modules/groups/commands/id.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { ID } from '@organicdesign/db-rpc-interfaces' -import { toString as uint8ArrayToString } from 'uint8arrays' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(ID.name, async (): Promise => { - return uint8ArrayToString(context.welo.identity.id, 'base58btc') - }) -} - -export default command diff --git a/packages/daemon/src/modules/groups/commands/join-group.ts b/packages/daemon/src/modules/groups/commands/join-group.ts deleted file mode 100644 index 7b8553a1..00000000 --- a/packages/daemon/src/modules/groups/commands/join-group.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { JoinGroup } from '@organicdesign/db-rpc-interfaces' -import { Address } from 'welo' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(JoinGroup.name, async (raw: unknown): Promise => { - const params = JoinGroup.Params.parse(raw) - const manifest = await context.welo.fetch(Address.fromString(`/hldb/${params.group}`)) - - try { - await context.groups.add(manifest) - } catch (error) { - if ((error as Error).message.includes('is already open')) { - throw new Error('group has already been joined') - } - - throw error - } - - return null - }) -} - -export default command diff --git a/packages/daemon/src/modules/groups/commands/list-groups.ts b/packages/daemon/src/modules/groups/commands/list-groups.ts deleted file mode 100644 index b6bdde3c..00000000 --- a/packages/daemon/src/modules/groups/commands/list-groups.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { ListGroups } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(ListGroups.name, async (): Promise => { - const promises: Array<{ group: string, name: string }> = [] - - for (const { key: cid, value: database } of context.groups.all()) { - promises.push({ group: cid, name: database.manifest.name }) - } - - return Promise.all(promises) - }) -} - -export default command diff --git a/packages/daemon/src/modules/groups/commands/sync.ts b/packages/daemon/src/modules/groups/commands/sync.ts deleted file mode 100644 index 6bc4a00c..00000000 --- a/packages/daemon/src/modules/groups/commands/sync.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Sync } from '@organicdesign/db-rpc-interfaces' -import { HeadsExchange } from 'welo/utils/heads-exchange' -import { cidstring } from 'welo/utils/index' -import { getHeads, addHeads } from 'welo/utils/replicator' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' -import type { Peer, Libp2p } from '@libp2p/interface' -import type { Database } from 'welo' - -const sync = async (libp2p: Libp2p, peer: Peer, database: Database, options: Partial<{ reverseSync: boolean, collisionRate: number, validate: boolean, rounds: number }> = {}): Promise => { - if (!await libp2p.peerStore.has(peer.id)) { - await libp2p.peerStore.save(peer.id, peer) - } - - // We need to dial so that libp2p can update multiaddrs. - await libp2p.dial(peer.id) - - const protocol = `/hldb/replicator/he/1.0.0/${cidstring(database.manifest.address.cid)}` - const stream = await libp2p.dialProtocol(peer.id, protocol) - const heads = await getHeads(database.replica) - const he = new HeadsExchange({ - stream, - heads, - remotePeerId: peer.id, - collisionRate: options.collisionRate ?? 0.1, - localPeerId: libp2p.peerId - }) - - // eslint-disable-next-line no-console - const pipePromise = he.pipe().catch(console.error) - - if (!(options.reverseSync ?? true)) { - await pipePromise - he.close() - await stream.close() - return - } - - try { - for (let i = 0; i < (options.rounds ?? 5); i++) { - const seed = i === 0 ? undefined : Math.floor(Math.random() * Number.MAX_SAFE_INTEGER) - const newHeads = await he.getHeads(seed) - - await addHeads(newHeads, database.replica, database.components) - - if (options.validate ?? true) { - const matches = await he.verify() - - if (matches) { - break - } - } - } - } catch (error) { - // Ignore errors. - } - - he.close() - await stream.close() -} - -const command: ModuleMethod = (context, { rpc, network }) => { - rpc.addMethod(Sync.name, async (): Promise => { - const peers = network.libp2p.getPeers() - const databases = context.welo.opened.values() - - const promises: Array> = [] - - for (const peerId of peers) { - const peer = await network.libp2p.peerStore.get(peerId) - - for (const database of databases) { - promises.push(sync(network.libp2p, peer, database)) - } - } - - await Promise.allSettled(promises) - - return null - }) -} - -export default command diff --git a/packages/daemon/src/modules/groups/entry-tracker.ts b/packages/daemon/src/modules/groups/entry-tracker.ts deleted file mode 100644 index 90f9e8b1..00000000 --- a/packages/daemon/src/modules/groups/entry-tracker.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { Key, type Datastore } from 'interface-datastore' -import { sha256 } from 'multiformats/hashes/sha2' -import { compare as uint8ArrayCompare } from 'uint8arrays/compare' -import type { KeyvalueDB, Pair } from '@/interface.js' -import { encodeAny, extendDatastore } from '@/utils.js' - -// Get the hash data from raw data. -const hashEntry = async (entry: unknown): Promise => { - const data = encodeAny(entry) - const digest = await sha256.digest(data) - - return digest.bytes -} - -// This class is responsible for keeping track of entries in the database. -export class EntryTracker { - private readonly datastore: Datastore - private readonly database: KeyvalueDB - - constructor (database: KeyvalueDB) { - this.datastore = extendDatastore(database.datastore, 'entry-tracker') - this.database = database - } - - async * process (prefix: string = '/'): AsyncGenerator> { - const index = await this.database.store.latest() - - for await (const { key, value } of index.query({ prefix })) { - const str = key.toString() - - if (await this.validate(str, value)) { - continue - } - - yield { key: key.toString(), value } - - await this.put(str, value) - } - } - - // Process an entry. - async put (key: string, entry: unknown): Promise { - const hash = await hashEntry(entry) - - await this.datastore.put(new Key(key), hash) - } - - async validate (key: string, entry: unknown): Promise { - const eHash = await this.getHash(key) - - if (eHash == null) { - return false - } - - const hash = await hashEntry(entry) - - return uint8ArrayCompare(eHash, hash) === 0 - } - - private async getHash (key: string): Promise { - try { - return await this.datastore.get(new Key(key)) - } catch (error) { - return null - } - } -} diff --git a/packages/daemon/src/modules/groups/groups.ts b/packages/daemon/src/modules/groups/groups.ts deleted file mode 100644 index 35e099c6..00000000 --- a/packages/daemon/src/modules/groups/groups.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { Key } from 'interface-datastore' -import { Manifest } from 'welo/manifest/index' -import { decodeCbor } from 'welo/utils/block' -import { logger } from './index.js' -import type { Pair, KeyvalueDB } from '@/interface.js' -import type { Startable } from '@libp2p/interfaces/startable' -import type { Datastore } from 'interface-datastore' -import type { CID } from 'multiformats/cid' -import type { Welo } from 'welo' -import type { ManifestData } from 'welo/manifest/interface' - -export interface Components { - datastore: Datastore - welo: Welo -} - -export class Groups implements Startable { - private readonly welo: Welo - private readonly datastore: Datastore - private readonly groups = new Map() - private started = false - - constructor (components: Components) { - this.welo = components.welo - this.datastore = components.datastore - } - - isStarted (): boolean { - return this.started - } - - async start (): Promise { - if (this.started) { - return - } - - for await (const pair of this.datastore.query({})) { - const block = await decodeCbor(pair.value) - const manifest = Manifest.asManifest({ block }) - - if (manifest == null) { - continue - } - - await this.add(manifest) - } - - this.started = true - } - - async stop (): Promise { - this.groups.clear() - this.started = false - } - - async add (manifest: Manifest): Promise { - const database = await this.welo.open(manifest) as KeyvalueDB - const address = manifest.address.cid.toString() - - this.groups.set(address, database) - - await this.datastore.put(new Key(database.address.cid.toString()), database.manifest.block.bytes) - - logger.info(`[groups] [join] ${manifest.address.cid.toString()}`) - } - - get (group: CID): KeyvalueDB | undefined { - return this.groups.get(group.toString()) - } - - * all (): Iterable> { - for (const [key, value] of this.groups.entries()) { - yield { key, value } - } - } -} - -export const createGroups = async (components: Components): Promise => { - const groups = new Groups(components) - - await groups.start() - - return groups -} diff --git a/packages/daemon/src/modules/groups/index.ts b/packages/daemon/src/modules/groups/index.ts deleted file mode 100644 index cdb8d99f..00000000 --- a/packages/daemon/src/modules/groups/index.ts +++ /dev/null @@ -1,53 +0,0 @@ -import createGroup from './commands/create-group.js' -import id from './commands/id.js' -import joinGroup from './commands/join-group.js' -import listGroups from './commands/list-groups.js' -import sync from './commands/sync.js' -import setupComponents from './setup.js' -import type { EntryTracker } from './entry-tracker.js' -import type { Groups } from './groups.js' -import type { Module, KeyvalueDB } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' -import type { Welo } from 'welo' -import { createLogger } from '@/logger.js' - -export const logger = createLogger('groups') - -export interface Requires extends Record { - base: Base - network: Network - rpc: RPC - sigint: Sigint -} - -export interface Provides extends Record { - welo: Welo - groups: Groups - getTracker(database: KeyvalueDB): EntryTracker -} - -const module: Module = async (components) => { - const context = await setupComponents(components) - - for (const setupCommand of [ - createGroup, - joinGroup, - listGroups, - sync, - id - ]) { - setupCommand(context, components) - } - - components.sigint.onInterupt(async () => { - await context.groups.stop() - await context.welo.stop() - }) - - return context -} - -export default module diff --git a/packages/daemon/src/modules/groups/setup.ts b/packages/daemon/src/modules/groups/setup.ts deleted file mode 100644 index f8025d2c..00000000 --- a/packages/daemon/src/modules/groups/setup.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' -import { EntryTracker } from './entry-tracker.js' -import { createGroups } from './groups.js' -import type { Requires, Provides } from './index.js' -import type { KeyvalueDB } from '@/interface.js' -import { extendDatastore } from '@/utils.js' - -export default async ({ base, network }: Requires): Promise => { - const welo = await createWelo({ - // @ts-expect-error Helia version mismatch here. - ipfs: network.helia, - replicators: [bootstrapReplicator(), pubsubReplicator()], - identity: await base.keyManager.getWeloIdentity() - }) - - const groups = await createGroups({ - datastore: extendDatastore(base.datastore, 'groups'), - welo - }) - - const getTracker = (database: KeyvalueDB): EntryTracker => new EntryTracker(database) - - return { - welo, - groups, - getTracker - } -} diff --git a/packages/daemon/src/modules/network/commands/addresses.ts b/packages/daemon/src/modules/network/commands/addresses.ts deleted file mode 100644 index 0bad66db..00000000 --- a/packages/daemon/src/modules/network/commands/addresses.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Addresses } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Addresses.name, async (): Promise => { - return context.libp2p.getMultiaddrs().map(a => a.toString()) - }) -} - -export default command diff --git a/packages/daemon/src/modules/network/commands/connect.ts b/packages/daemon/src/modules/network/commands/connect.ts deleted file mode 100644 index f404843f..00000000 --- a/packages/daemon/src/modules/network/commands/connect.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { multiaddr } from '@multiformats/multiaddr' -import { Connect } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Connect.name, async (raw: unknown): Promise => { - const params = Connect.Params.parse(raw) - const address = multiaddr(params.address) - - await context.libp2p.dial(address) - - return null - }) -} - -export default command diff --git a/packages/daemon/src/modules/network/commands/connections.ts b/packages/daemon/src/modules/network/commands/connections.ts deleted file mode 100644 index 34ba86dd..00000000 --- a/packages/daemon/src/modules/network/commands/connections.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Connections } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Connections.name, async (): Promise => { - return context.libp2p.getConnections().map(c => c.remoteAddr.toString()) - }) -} - -export default command diff --git a/packages/daemon/src/modules/network/commands/count-peers.ts b/packages/daemon/src/modules/network/commands/count-peers.ts deleted file mode 100644 index 3966b7d5..00000000 --- a/packages/daemon/src/modules/network/commands/count-peers.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { CountPeers } from '@organicdesign/db-rpc-interfaces' -import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(CountPeers.name, async (raw: unknown): Promise => { - const countPeers = async (cid: CID, options?: { timeout: number }): Promise => { - let count = 0 - - const itr = context.libp2p.contentRouting.findProviders(cid, { - signal: AbortSignal.timeout(options?.timeout ?? 3000) - }) - - try { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const _ of itr) { - count++ - } - } catch (error) { - // Do nothing - } - - return count - } - - const params = CountPeers.Params.parse(raw) - const cids = params.cids.map(cid => CID.parse(cid)) - - return Promise.all(cids.map(async cid => ({ - cid: cid.toString(), - peers: await countPeers(cid) - }))) - }) -} - -export default command diff --git a/packages/daemon/src/modules/network/index.ts b/packages/daemon/src/modules/network/index.ts deleted file mode 100644 index 4966539d..00000000 --- a/packages/daemon/src/modules/network/index.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { z } from 'zod' -import addresses from './commands/addresses.js' -import connect from './commands/connect.js' -import connections from './commands/connections.js' -import countPeers from './commands/count-peers.js' -import setupComponents from './setup.js' -import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' -import type { Libp2p } from '@libp2p/interface' -import type PinManager from '@organicdesign/db-helia-pin-manager' -import type { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' -import type { Helia } from 'helia' -import { createLogger } from '@/logger.js' - -export const logger = createLogger('network') - -const Config = z.object({ - serverMode: z.boolean().default(false), - private: z.boolean().default(false), - bootstrap: z.array(z.string()).default([]), - addresses: z.array(z.string()).default([ - '/ip4/127.0.0.1/tcp/0', - '/ip4/127.0.0.1/tcp/0/ws' - ]) -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type Config = z.output - -export interface Requires extends Record { - base: Base - rpc: RPC - config: ConfigModule - sigint: Sigint -} - -export interface Provides extends Record { - libp2p: Libp2p - helia: Helia - pinManager: PinManager - config: Config - manualBlockBroker: ManualBlockBroker -} - -const module: Module = async (components) => { - const config = components.config.get(Config) - - const context = await setupComponents(components, config) - - for (const setupCommand of [addresses, connect, connections, countPeers]) { - setupCommand(context, components) - } - - components.sigint.onInterupt(async () => { - await context.helia.stop() - await context.libp2p.stop() - }) - - return context -} - -export default module diff --git a/packages/daemon/src/modules/network/libp2p.ts b/packages/daemon/src/modules/network/libp2p.ts deleted file mode 100644 index 5b472891..00000000 --- a/packages/daemon/src/modules/network/libp2p.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { gossipsub } from '@chainsafe/libp2p-gossipsub' -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { autoNAT } from '@libp2p/autonat' -import { bootstrap } from '@libp2p/bootstrap' -import { circuitRelayTransport, circuitRelayServer } from '@libp2p/circuit-relay-v2' -import { dcutr } from '@libp2p/dcutr' -import { identify } from '@libp2p/identify' -import { kadDHT, removePrivateAddressesMapper, removePublicAddressesMapper } from '@libp2p/kad-dht' -import { preSharedKey } from '@libp2p/pnet' -import { tcp } from '@libp2p/tcp' -import { uPnPNAT } from '@libp2p/upnp-nat' -import { webSockets } from '@libp2p/websockets' -import { createLibp2p } from 'libp2p' -import type { Libp2p, PeerId } from '@libp2p/interface' -import type { Datastore } from 'interface-datastore' - -export default async ({ datastore, peerId, psk, addresses, bootstrap: bs, serverMode }: { datastore?: Datastore, peerId?: PeerId, psk?: Uint8Array, addresses?: string[], bootstrap?: string[], serverMode?: boolean }): Promise => { - const services: Record = {} - - if (serverMode === true) { - services.circuitRelay = circuitRelayServer() - } - - return createLibp2p({ - peerId, - datastore, - transports: [tcp(), webSockets(), circuitRelayTransport({ discoverRelays: 2 })], - connectionEncryption: [noise()], - streamMuxers: [yamux()], - connectionProtector: (psk != null) ? preSharedKey({ psk }) : undefined, - - addresses: { - listen: addresses ?? [ - '/ip4/127.0.0.1/tcp/0', - '/ip4/127.0.0.1/tcp/0/ws' - // "/ip6/::/tcp/0" - ] - }, - - connectionManager: { - autoDialInterval: 6e3 - }, - - services: { - ...services, - identify: identify(), - pubsub: gossipsub({ allowPublishToZeroPeers: true }), - autoNAT: autoNAT(), - dcutr: dcutr(), - upnpNAT: uPnPNAT(), - - dht: kadDHT({ - protocol: '/ipfs/kad/1.0.0', - peerInfoMapper: removePrivateAddressesMapper - }), - - lanDHT: kadDHT({ - protocol: '/ipfs/lan/kad/1.0.0', - peerInfoMapper: removePublicAddressesMapper, - clientMode: false - }) - }, - - peerDiscovery: (bs != null) && bs.length > 0 - ? [ - bootstrap({ - list: bs ?? [] - }) - ] - : [] - }) -} diff --git a/packages/daemon/src/modules/network/setup.ts b/packages/daemon/src/modules/network/setup.ts deleted file mode 100644 index 0582cd11..00000000 --- a/packages/daemon/src/modules/network/setup.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { bitswap } from '@helia/block-brokers' -import PinManager from '@organicdesign/db-helia-pin-manager' -import { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' -import { createHelia } from 'helia' -import createLibp2p from './libp2p.js' -import { type Requires, type Provides, type Config, logger } from './index.js' -import { extendDatastore } from '@/utils.js' - -export default async ({ base }: Requires, config: Config): Promise => { - const peerId = await base.keyManager.getPeerId() - const psk = base.keyManager.getPskKey() - - const libp2p = await createLibp2p({ - datastore: extendDatastore(base.datastore, 'datastore/libp2p'), - psk: config.private ? psk : undefined, - peerId, - ...config - }) - - const manualBlockBroker = new ManualBlockBroker() - - const helia = await createHelia({ - datastore: extendDatastore(base.datastore, 'helia/datastore'), - libp2p, - blockstore: base.blockstore, - blockBrokers: [bitswap(), () => manualBlockBroker] - }) - - const pinManager = new PinManager({ - helia, - datastore: extendDatastore(base.datastore, 'pinManager') - }) - - pinManager.events.addEventListener('downloads:added', ({ cid }) => { - logger.info(`[downloads] [+] ${cid}`) - }) - - pinManager.events.addEventListener('pins:added', ({ cid }) => { - logger.info(`[pins] [+] ${cid}`) - }) - - pinManager.events.addEventListener('pins:adding', ({ cid }) => { - logger.info(`[pins] [~] ${cid}`) - }) - - pinManager.events.addEventListener('pins:removed', ({ cid }) => { - logger.info(`[pins] [-] ${cid}`) - }) - - return { - libp2p, - helia, - pinManager, - config, - manualBlockBroker - } -} diff --git a/packages/daemon/src/modules/rpc/index.ts b/packages/daemon/src/modules/rpc/index.ts deleted file mode 100644 index cf346a19..00000000 --- a/packages/daemon/src/modules/rpc/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { createNetServer } from '@organicdesign/net-rpc' -import type { Module } from '@/interface.js' -import type { Provides as Argv } from '@/modules/argv/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' - -export interface Requires extends Record { - argv: Argv - sigint: Sigint -} - -export interface Provides extends Record { - addMethod (name: string, method: (arg: unknown) => any): void -} - -const module: Module = async (components) => { - const { rpc, close } = await createNetServer(components.argv.socket) - - const addMethod = (name: string, method: (arg: unknown) => any): void => { - rpc.addMethod(name, method) - } - - components.sigint.onInterupt(close) - - return { addMethod } -} - -export default module diff --git a/packages/daemon/src/modules/sigint/index.ts b/packages/daemon/src/modules/sigint/index.ts deleted file mode 100644 index a2d188da..00000000 --- a/packages/daemon/src/modules/sigint/index.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { Module } from '@/interface.js' -import { createLogger } from '@/logger.js' - -export const logger = createLogger('sigint') - -export interface Provides extends Record { - onInterupt (method: () => unknown): void - interupt (): Promise -} - -const module: Module = async () => { - const methods: Array<() => unknown> = [] - - let exiting = false - - const interupt = async (): Promise => { - if (exiting) { - logger.warn('force exiting') - process.exit(1) - } - - exiting = true - - logger.info('cleaning up...') - - for (const method of methods) { - try { - await method() - } catch (error) { - logger.error(error) - } - } - - logger.info('exiting...') - } - - return { - onInterupt: (method) => methods.push(method), - interupt - } -} - -export default module diff --git a/packages/daemon/src/modules/sneakernet/commands/sneakernet-receive.ts b/packages/daemon/src/modules/sneakernet/commands/sneakernet-receive.ts deleted file mode 100644 index 30dd8041..00000000 --- a/packages/daemon/src/modules/sneakernet/commands/sneakernet-receive.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { SneakernetReveive } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(SneakernetReveive.name, async (raw: unknown): Promise => { - const params = SneakernetReveive.Params.parse(raw) - - await context.sneakernet.import(params.path) - - return null - }) -} - -export default command diff --git a/packages/daemon/src/modules/sneakernet/commands/sneakernet-send.ts b/packages/daemon/src/modules/sneakernet/commands/sneakernet-send.ts deleted file mode 100644 index 17ebfb21..00000000 --- a/packages/daemon/src/modules/sneakernet/commands/sneakernet-send.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { SneakernetSend } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(SneakernetSend.name, async (raw: unknown): Promise => { - const params = SneakernetSend.Params.parse(raw) - - await context.sneakernet.export(params.path, params.peers) - - return null - }) -} - -export default command diff --git a/packages/daemon/src/modules/sneakernet/index.ts b/packages/daemon/src/modules/sneakernet/index.ts deleted file mode 100644 index 9a50795d..00000000 --- a/packages/daemon/src/modules/sneakernet/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { RevisionStrategies } from '@organicdesign/db-rpc-interfaces/zod' -import { z } from 'zod' -import setupSneakernetReceive from './commands/sneakernet-receive.js' -import setupSneakernetSend from './commands/sneakernet-send.js' -import { Sneakernet } from './sneakernet.js' -import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as Groups } from '@/modules/groups/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import { createLogger } from '@/logger.js' -import { extendDatastore } from '@/utils.js' - -export const logger = createLogger('sneakernet') - -export const Config = z.object({ - defaultRevisionStrategy: RevisionStrategies.default('all') -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type Config = z.output - -export interface Requires extends Record { - groups: Groups - base: Base - rpc: RPC - network: Network -} - -export interface Provides extends Record { - sneakernet: Sneakernet -} - -const module: Module = async (components) => { - // This module needs to be improved once the next gen Welo is released. - // It only transfers missing blocks but if we could workout difference between - // heads we could pre-emptively transfer missing data. - const datastore = extendDatastore(components.base.datastore, 'sneakernet') - const sneakernet = new Sneakernet(components, datastore) - const context = { sneakernet } - - for (const setupCommand of [setupSneakernetSend, setupSneakernetReceive]) { - setupCommand(context, components) - } - - return context -} - -export default module diff --git a/packages/daemon/src/modules/sneakernet/interface.ts b/packages/daemon/src/modules/sneakernet/interface.ts deleted file mode 100644 index 69061fd2..00000000 --- a/packages/daemon/src/modules/sneakernet/interface.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { z } from 'zod' - -export const PeerData = z.object({ - // This is a list of blocks that this peer wants. - wants: z.array(z.instanceof(Uint8Array)), - - // This is a list of the heads this peer is known to have. - heads: z.array(z.object({ - group: z.instanceof(Uint8Array), - cids: z.array(z.instanceof(Uint8Array)) - })) -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type PeerData = z.infer - -export const EncodedPeerData = z.array(z.object({ - id: z.instanceof(Uint8Array) -}).extend(PeerData.shape)) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type EncodedPeerData = z.infer diff --git a/packages/daemon/src/modules/sneakernet/sneakernet.ts b/packages/daemon/src/modules/sneakernet/sneakernet.ts deleted file mode 100644 index 2b453e8c..00000000 --- a/packages/daemon/src/modules/sneakernet/sneakernet.ts +++ /dev/null @@ -1,135 +0,0 @@ -import fss from 'fs' -import fs from 'fs/promises' -import Path from 'path' -import { Readable } from 'stream' -import { car, type Car } from '@helia/car' -import { CarWriter, CarReader } from '@ipld/car' -import * as cborg from 'cborg' -import { type Datastore, Key } from 'interface-datastore' -import all from 'it-all' -import { CID } from 'multiformats/cid' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { getHeads } from 'welo/utils/replicator' -import { EncodedPeerData, PeerData } from './interface.js' -import type { Requires } from './index.js' -import type { KeyvalueDB, Pair } from '@/interface.js' -import type { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' -import type { Blockstore } from 'interface-blockstore' - -export class Sneakernet { - private readonly datastore: Datastore - private readonly blockstore: Blockstore - private readonly id: Uint8Array - private readonly broker: ManualBlockBroker - private readonly getGroups: () => Array> - private readonly car: Car - - constructor (components: Requires, datastore: Datastore) { - this.datastore = datastore - this.id = components.groups.welo.identity.id - this.broker = components.network.manualBlockBroker - this.getGroups = () => [...components.groups.groups.all()] - this.car = car(components.network.helia) - this.blockstore = components.base.blockstore - } - - async export (path: string, peers: string[] = []): Promise { - await Promise.all([ - this.exportManifest(Path.join(path, 'manifest')), - this.exportBlocks(Path.join(path, 'blocks.car'), peers) - ]) - } - - async import (path: string): Promise { - await Promise.all([ - this.importManifest(Path.join(path, 'manifest')), - this.importBlocks(Path.join(path, 'blocks.car')) - ]) - } - - private async importManifest (path: string): Promise { - const rawData = await fs.readFile(path) - const data = EncodedPeerData.parse(cborg.decode(rawData)) - - await all(this.datastore.putMany(data.map(peerData => ({ - key: new Key(peerData.id), - value: cborg.encode({ - wants: peerData.wants, - heads: peerData.heads - }) - })))) - } - - private async importBlocks (path: string): Promise { - try { - await fs.stat(path) - } catch (error) { - // No blocks file - just return. - return - } - - const inStream = fss.createReadStream(path) - const reader = await CarReader.fromIterable(inStream) - - await this.car.import(reader) - } - - private async exportManifest (path: string): Promise { - const pairs = await Promise.all( - this.getGroups().map(async ({ value: database }) => ({ - group: database.address.cid.bytes, - cids: (await getHeads(database.replica)).map(h => h.bytes) - })) - ) - - const peerData: EncodedPeerData = [{ - id: this.id, - wants: [...this.broker.getWants()].map(c => c.bytes), - heads: pairs - }] - - const data = cborg.encode(peerData) - - await fs.writeFile(path, data) - } - - private async exportBlocks (path: string, peers: string[] = []): Promise { - if (peers.length === 0) { - return - } - - const blocks: CID[] = [] - - const addWants = async (wants: Uint8Array[]): Promise => { - for (const cidData of wants) { - const cid = CID.decode(cidData) - - if (await this.blockstore.has(cid)) { - blocks.push(cid) - } - } - } - - for (const peer of peers) { - try { - const key = new Key(uint8ArrayFromString(peer, 'base58btc')) - const data = await this.datastore.get(key) - const peerData = PeerData.parse(cborg.decode(data)) - - await addWants(peerData.wants) - } catch (error) { - // Ignore - } - } - - if (blocks.length === 0) { - return - } - - const { writer, out } = await CarWriter.create(blocks) - - Readable.from(out).pipe(fss.createWriteStream(path)) - - await this.car.export(blocks, writer) - } -} diff --git a/packages/daemon/src/modules/tick/index.ts b/packages/daemon/src/modules/tick/index.ts deleted file mode 100644 index 578126b6..00000000 --- a/packages/daemon/src/modules/tick/index.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { z } from 'zod' -import type { Module } from '@/interface.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' -import { createLogger } from '@/logger.js' - -export const logger = createLogger('tick') - -const Config = z.object({ - tickInterval: z.number().default(10 * 60) -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type Config = z.output - -export interface Requires extends Record { - config: ConfigModule - sigint: Sigint -} - -export interface Provides extends Record { - config: Config - register (method: (...args: any[]) => any): void -} - -const module: Module = async (components) => { - const config = components.config.get(Config) - const methods: Array<(...args: any[]) => any> = [] - const register = (method: (...args: any[]) => any): void => { methods.push(method) } - - let stopping = false - - components.sigint.onInterupt(() => { - stopping = true - }) - - void (async () => { - for (;;) { - for (const method of methods) { - try { - await method() - } catch (error) { - logger.warn('method threw: ', error) - } - - if (stopping) { - return true - } - } - - const b = await new Promise(resolve => { - const timeout = setTimeout(() => { resolve(false) }, config.tickInterval) - - components.sigint.onInterupt(() => { - if (timeout != null) { - clearTimeout(timeout) - } - - resolve(true) - }) - }) - - if (b) { - return - } - } - })() - - return { config, register } -} - -export default module From c76665857562082ab37d7ac8da377ef82b6bb863 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:45:25 +1300 Subject: [PATCH 25/62] Linting. --- packages/daemon/src/common/downloader/index.ts | 6 +++--- .../daemon/src/common/pin-manager/{utils.js => utils.ts} | 0 packages/daemon/src/common/tick.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename packages/daemon/src/common/pin-manager/{utils.js => utils.ts} (100%) diff --git a/packages/daemon/src/common/downloader/index.ts b/packages/daemon/src/common/downloader/index.ts index bab0070e..af22abf3 100644 --- a/packages/daemon/src/common/downloader/index.ts +++ b/packages/daemon/src/common/downloader/index.ts @@ -29,18 +29,18 @@ export class Downloader implements Startable { this.pinManager = pinManager } - async start () { + async start (): Promise { await this.loopPromise this.controller = new AbortController() this.loopPromise = this.loop() } - async stop () { + async stop (): Promise { this.controller.abort() await this.loopPromise } - private async loop () { + private async loop (): Promise { for (;;) { if (this.isAborted) { return diff --git a/packages/daemon/src/common/pin-manager/utils.js b/packages/daemon/src/common/pin-manager/utils.ts similarity index 100% rename from packages/daemon/src/common/pin-manager/utils.js rename to packages/daemon/src/common/pin-manager/utils.ts diff --git a/packages/daemon/src/common/tick.ts b/packages/daemon/src/common/tick.ts index 7041028e..3e00ae01 100644 --- a/packages/daemon/src/common/tick.ts +++ b/packages/daemon/src/common/tick.ts @@ -65,7 +65,7 @@ export class Tick implements Startable { } await new Promise(resolve => { - const listener = () => { + const listener = (): void => { if (timeout != null) { clearTimeout(timeout) this.signal.removeEventListener('abort', listener) From 204cf19202fa76a41cab2677c0e77651b0fdb310 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:47:32 +1300 Subject: [PATCH 26/62] Fix other filesystem command. --- packages/daemon/src/modules/filesystem/commands/edit.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/daemon/src/modules/filesystem/commands/edit.ts b/packages/daemon/src/modules/filesystem/commands/edit.ts index 88bad302..cf9e36bd 100644 --- a/packages/daemon/src/modules/filesystem/commands/edit.ts +++ b/packages/daemon/src/modules/filesystem/commands/edit.ts @@ -1,10 +1,10 @@ import { Edit } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import { type Provides, type Requires, logger } from '../index.js' +import { type Context, logger } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Edit.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, context) => { + net.rpc.addMethod(Edit.name, async (raw: unknown): Promise => { const params = Edit.Params.parse(raw) if (params.revisionStrategy !== null) { From 02e2fc4d2074504cc4a98c61272c384c65388e12 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 16:56:56 +1300 Subject: [PATCH 27/62] Fix events. --- packages/daemon/src/common/index.ts | 4 +++- packages/daemon/src/common/interface.ts | 1 + .../src/modules/filesystem/commands/write.ts | 5 ++--- .../daemon/src/modules/filesystem/events.ts | 21 ------------------- .../daemon/src/modules/filesystem/index.ts | 4 +--- .../daemon/src/modules/filesystem/setup.ts | 8 ++----- .../modules/filesystem/upload-operations.ts | 5 ++--- .../daemon/src/modules/revisions/setup.ts | 12 ++++++++--- 8 files changed, 20 insertions(+), 40 deletions(-) delete mode 100644 packages/daemon/src/modules/filesystem/events.ts diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 54bd5566..a85ef880 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -32,6 +32,7 @@ export default async (): Promise => { const controller = new AbortController() const net = await createNetServer(argv.socket) const config = getConfig(Config) + const events = new EventTarget() const datastore = isMemory(config.storage) ? new MemoryDatastore() @@ -157,7 +158,8 @@ export default async (): Promise => { groups, pinManager, welo, - heliaPinManager + heliaPinManager, + events } return components diff --git a/packages/daemon/src/common/interface.ts b/packages/daemon/src/common/interface.ts index 37707890..e1127b5f 100644 --- a/packages/daemon/src/common/interface.ts +++ b/packages/daemon/src/common/interface.ts @@ -46,4 +46,5 @@ export interface Components { groups: Groups pinManager: PinManager heliaPinManager: HeliaPinManager + events: EventTarget } diff --git a/packages/daemon/src/modules/filesystem/commands/write.ts b/packages/daemon/src/modules/filesystem/commands/write.ts index b7452d52..6f97b63c 100644 --- a/packages/daemon/src/modules/filesystem/commands/write.ts +++ b/packages/daemon/src/modules/filesystem/commands/write.ts @@ -4,12 +4,11 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { FileSystemEvent } from '../events.js' import type { Context } from '../index.js' import type { Entry } from '../interface.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = ({ net, helia }, context) => { +const command: ModuleMethod = ({ net, helia, events }, context) => { net.rpc.addMethod(Write.name, async (raw: unknown): Promise => { const params = Write.Params.parse(raw) const group = CID.parse(params.group) @@ -41,7 +40,7 @@ const command: ModuleMethod = ({ net, helia }, context) => { const newEntry = await fs.put(params.path, newEntryParams) - context.events.dispatchEvent(new FileSystemEvent('file:added', group, params.path, newEntry)) + events.dispatchEvent(new CustomEvent('file:added', { detail: { group, path: params.path, entry: newEntry } })) return params.data.length }) diff --git a/packages/daemon/src/modules/filesystem/events.ts b/packages/daemon/src/modules/filesystem/events.ts deleted file mode 100644 index b8c99688..00000000 --- a/packages/daemon/src/modules/filesystem/events.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Event, type EventTarget } from 'ts-event-target' -import type { Entry } from './interface.js' -import type { CID } from 'multiformats/cid' - -export type EventTypes = 'file:added' - -export class FileSystemEvent extends Event { - readonly entry: Entry - readonly path: string - readonly group: CID - - constructor (type: EventTypes, group: CID, path: string, entry: Entry) { - super(type) - - this.entry = entry - this.path = path - this.group = group - } -} - -export type Events = EventTarget<[FileSystemEvent]> diff --git a/packages/daemon/src/modules/filesystem/index.ts b/packages/daemon/src/modules/filesystem/index.ts index b47e4796..47e10705 100644 --- a/packages/daemon/src/modules/filesystem/index.ts +++ b/packages/daemon/src/modules/filesystem/index.ts @@ -10,7 +10,6 @@ import write from './commands/write.js' import { type FileSystem } from './file-system.js' import setup from './setup.js' import syncGroups from './sync-groups.js' -import type { Events } from './events.js' import type { LocalSettings } from './local-settings.js' import type createUploadManager from './upload-operations.js' import type { Module } from '@/interface.js' @@ -31,7 +30,6 @@ export interface Context extends Record { localSettings: LocalSettings config: Config getFileSystem (group: CID): FileSystem | null - events: Events } const module: Module = async (components) => { @@ -47,7 +45,7 @@ const module: Module = async (components) => { read, write ]) { - setupCommand(context, components) + setupCommand(components, context) } components.tick.add(async () => syncGroups(components, context)) diff --git a/packages/daemon/src/modules/filesystem/setup.ts b/packages/daemon/src/modules/filesystem/setup.ts index 73023461..b6a1d26a 100644 --- a/packages/daemon/src/modules/filesystem/setup.ts +++ b/packages/daemon/src/modules/filesystem/setup.ts @@ -1,8 +1,6 @@ -import { EventTarget } from 'ts-event-target' import { FileSystem } from './file-system.js' import { LocalSettings } from './local-settings.js' import createUploadManager from './upload-operations.js' -import type { Events } from './events.js' import type { Context, Config } from './index.js' import type { Components } from '@/common/interface.js' import type { CID } from 'multiformats/cid' @@ -10,7 +8,6 @@ import { extendDatastore } from '@/utils.js' export default async (components: Components, config: Config): Promise => { const { groups, datastore, blockstore, welo } = components - const events: Events = new EventTarget() const localSettings = new LocalSettings({ datastore: extendDatastore(datastore, 'references') @@ -32,7 +29,7 @@ export default async (components: Components, config: Config): Promise } const uploads = await createUploadManager( - { getFileSystem, events }, + { getFileSystem }, components, extendDatastore(datastore, 'upload-operations') ) @@ -41,7 +38,6 @@ export default async (components: Components, config: Config): Promise localSettings, uploads, config, - getFileSystem, - events + getFileSystem } } diff --git a/packages/daemon/src/modules/filesystem/upload-operations.ts b/packages/daemon/src/modules/filesystem/upload-operations.ts index ef1a36ff..c2706512 100644 --- a/packages/daemon/src/modules/filesystem/upload-operations.ts +++ b/packages/daemon/src/modules/filesystem/upload-operations.ts @@ -4,7 +4,6 @@ import { type RevisionStrategies } from '@organicdesign/db-rpc-interfaces/zod' import all from 'it-all' import { CID } from 'multiformats/cid' import { take } from 'streaming-iterables' -import { FileSystemEvent } from './events.js' import type { Context } from './index.js' import type { Entry } from './interface.js' import type { Components } from '@/common/interface.js' @@ -12,7 +11,7 @@ import type { Pair } from '@/interface.js' import type { Datastore } from 'interface-datastore' import { OperationManager } from '@/operation-manager.js' -export default async (context: Pick, { pinManager, helia }: Components, datastore: Datastore): Promise, { events, pinManager, helia }: Components, datastore: Datastore): Promise delete(groupData: Uint8Array, path: string): Promise>> }>> => { @@ -28,7 +27,7 @@ export default async (context: Pick, { pinM const fullEntry = await fs.put(path, { ...entry, cid }) await pinManager.put(path, { cid, priority: entry.priority }) - context.events.dispatchEvent(new FileSystemEvent('file:added', group, path, fullEntry)) + events.dispatchEvent(new CustomEvent('file:added', { detail: { group, path, entry: fullEntry } })) } const om = new OperationManager(datastore, { diff --git a/packages/daemon/src/modules/revisions/setup.ts b/packages/daemon/src/modules/revisions/setup.ts index a75f3b7d..cf94b637 100644 --- a/packages/daemon/src/modules/revisions/setup.ts +++ b/packages/daemon/src/modules/revisions/setup.ts @@ -6,7 +6,7 @@ import { type Context, logger } from './index.js' import type { Components } from '@/common/interface.js' import type { CID } from 'multiformats/cid' -export default async ({ groups, welo }: Components): Promise => { +export default async ({ groups, welo, events }: Components): Promise => { const getRevisions = (group: CID): Revisions | null => { const database = groups.get(group) @@ -17,8 +17,14 @@ export default async ({ groups, welo }: Components): Promise => { return new Revisions(database, welo.identity.id) } - filesystem.events.addEventListener('file:added', ({ group, path, entry }) => { - (async () => { + events.addEventListener('file:added', (event) => { + if (!(event instanceof CustomEvent)) { + return + } + + const { detail: { group, path, entry } } = event + + ;(async () => { const revisions = getRevisions(group) if (revisions == null) { From 21380bd9a65265cddfb08b57bd0e6ce07503e8e1 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 17:01:31 +1300 Subject: [PATCH 28/62] Fix revisions module. --- .../src/modules/revisions/commands/export-revision.ts | 6 ++---- .../src/modules/revisions/commands/read-revision.ts | 8 ++++---- packages/daemon/src/modules/revisions/index.ts | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/daemon/src/modules/revisions/commands/export-revision.ts b/packages/daemon/src/modules/revisions/commands/export-revision.ts index d2567d79..ec28a88e 100644 --- a/packages/daemon/src/modules/revisions/commands/export-revision.ts +++ b/packages/daemon/src/modules/revisions/commands/export-revision.ts @@ -11,16 +11,14 @@ const command: ModuleMethod = ({ net, blockstore }, context) => { const params = ExportRevision.Params.parse(raw) const group = CID.parse(params.group) const revisions = context.getRevisions(group) - const fs = filesystem.getFileSystem(group) - if (revisions == null || fs == null) { + if (revisions == null) { throw new Error('no such group') } const author = uint8ArrayFromString(params.author, 'base58btc') - for await (const pair of fs.getDir(params.path)) { - const path = pair.key.toString() + for await (const { path } of revisions.getAll(params.path)) { const outFile = Path.join(params.outPath, path.replace(params.path, '')) const revision = await revisions.get(path, author, params.sequence) diff --git a/packages/daemon/src/modules/revisions/commands/read-revision.ts b/packages/daemon/src/modules/revisions/commands/read-revision.ts index adc38295..e0135225 100644 --- a/packages/daemon/src/modules/revisions/commands/read-revision.ts +++ b/packages/daemon/src/modules/revisions/commands/read-revision.ts @@ -5,11 +5,11 @@ import { collect } from 'streaming-iterables' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, network }) => { - rpc.addMethod(ReadRevision.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, helia }, context) => { + net.rpc.addMethod(ReadRevision.name, async (raw: unknown): Promise => { const params = ReadRevision.Params.parse(raw) const group = CID.parse(params.group) const revisions = context.getRevisions(group) @@ -26,7 +26,7 @@ const command: ModuleMethod = (context, { rpc, network }) => throw new Error(`no such revision: ${params.path}, ${author}, ${params.sequence}`) } - const ufs = unixfs(network.helia) + const ufs = unixfs(helia) return uint8ArrayToString(uint8ArrayConcat(await collect(ufs.cat(entry.cid, { offset: params.position, length: params.length })))) }) diff --git a/packages/daemon/src/modules/revisions/index.ts b/packages/daemon/src/modules/revisions/index.ts index e9ef069a..22e10821 100644 --- a/packages/daemon/src/modules/revisions/index.ts +++ b/packages/daemon/src/modules/revisions/index.ts @@ -27,7 +27,7 @@ const module: Module = async (components) => { const context = await setup(components) for (const setupCommand of [listRevisions, exportRevision, readRevision]) { - setupCommand(context, components) + setupCommand(components, context) } components.tick.add(async () => syncRevisions(components)) From 7baa79dd0a3c52d6b7c591194057319ebaa23a82 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 27 Mar 2024 17:09:12 +1300 Subject: [PATCH 29/62] Setup common commands. --- packages/daemon/src/common/handle-commands.ts | 38 +++++++++++++++++++ packages/daemon/src/common/index.ts | 3 ++ 2 files changed, 41 insertions(+) create mode 100644 packages/daemon/src/common/handle-commands.ts diff --git a/packages/daemon/src/common/handle-commands.ts b/packages/daemon/src/common/handle-commands.ts new file mode 100644 index 00000000..e6a9f20d --- /dev/null +++ b/packages/daemon/src/common/handle-commands.ts @@ -0,0 +1,38 @@ +import addresses from './commands/addresses.js' +import connect from './commands/connect.js' +import connections from './commands/connections.js' +import countPeers from './commands/count-peers.js' +import createGroup from './commands/create-group.js' +import getSpeeds from './commands/get-speeds.js' +import getStatus from './commands/get-status.js' +import id from './commands/id.js' +import joinGroup from './commands/join-group.js' +import listGroups from './commands/list-groups.js' +import setPriority from './commands/set-priority.js' +import sneakernetReveive from './commands/sneakernet-receive.js' +import sneakernetSend from './commands/sneakernet-send.js' +import sync from './commands/sync.js' +import type { Components } from './interface.js' + +export default (components: Components): void => { + const commands = [ + addresses, + connect, + connections, + countPeers, + createGroup, + getSpeeds, + getStatus, + id, + joinGroup, + listGroups, + setPriority, + sneakernetReveive, + sneakernetSend, + sync + ] + + for (const command of commands) { + command(components, {}) + } +} diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index a85ef880..e53ca7a4 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -13,6 +13,7 @@ import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' import { createDownloader } from './downloader/index.js' import { EntryTracker } from './entry-tracker.js' import { createGroups } from './groups.js' +import handleCommands from './handle-commands.js' import { Config, type Components } from './interface.js' import createLibp2p from './libp2p.js' import parseArgv from './parse-argv.js' @@ -162,5 +163,7 @@ export default async (): Promise => { events } + handleCommands(components) + return components } From 139f7c12265f772eb4711fb8f5ca716558cee77f Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 09:52:22 +1300 Subject: [PATCH 30/62] Fix config and welo start. --- packages/daemon/src/common/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index e53ca7a4..de319d00 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -28,7 +28,7 @@ import { isMemory, extendDatastore } from '@/utils.js' export default async (): Promise => { const argv = await parseArgv() const logger = createLogger('common') - const getConfig = await parseConfig(argv.socket) + const getConfig = await parseConfig(argv.config) const keyManager = await createKeyManager(argv.key) const controller = new AbortController() const net = await createNetServer(argv.socket) @@ -64,7 +64,7 @@ export default async (): Promise => { const welo = await createWelo({ // @ts-expect-error Helia version mismatch here. - ipfs: network.helia, + ipfs: helia, replicators: [bootstrapReplicator(), pubsubReplicator()], identity: await keyManager.getWeloIdentity() }) @@ -131,7 +131,9 @@ export default async (): Promise => { }) controller.signal.addEventListener('abort', () => { - (async () => { + logger.info('exiting...') + + ;(async () => { await net.close() await tick.stop() await downloader.stop() From bb7500b01b74a0c541c0fedff81519db4d1ee2e0 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 12:18:25 +1300 Subject: [PATCH 31/62] Fix events. --- packages/daemon/src/common/index.ts | 3 ++- packages/daemon/src/modules/filesystem/commands/write.ts | 1 + packages/daemon/src/modules/filesystem/upload-operations.ts | 1 + packages/daemon/src/modules/revisions/setup.ts | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index de319d00..52efe390 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -131,7 +131,7 @@ export default async (): Promise => { }) controller.signal.addEventListener('abort', () => { - logger.info('exiting...') + logger.info('cleaning up...') ;(async () => { await net.close() @@ -141,6 +141,7 @@ export default async (): Promise => { await welo.stop() await helia.stop() await libp2p.stop() + logger.info('exiting...') })().catch(error => { logger.error(error) }) diff --git a/packages/daemon/src/modules/filesystem/commands/write.ts b/packages/daemon/src/modules/filesystem/commands/write.ts index 6f97b63c..f3506f5f 100644 --- a/packages/daemon/src/modules/filesystem/commands/write.ts +++ b/packages/daemon/src/modules/filesystem/commands/write.ts @@ -7,6 +7,7 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import type { Context } from '../index.js' import type { Entry } from '../interface.js' import type { ModuleMethod } from '@/interface.js' +import { CustomEvent } from '@libp2p/interface' const command: ModuleMethod = ({ net, helia, events }, context) => { net.rpc.addMethod(Write.name, async (raw: unknown): Promise => { diff --git a/packages/daemon/src/modules/filesystem/upload-operations.ts b/packages/daemon/src/modules/filesystem/upload-operations.ts index c2706512..fcd63c2c 100644 --- a/packages/daemon/src/modules/filesystem/upload-operations.ts +++ b/packages/daemon/src/modules/filesystem/upload-operations.ts @@ -10,6 +10,7 @@ import type { Components } from '@/common/interface.js' import type { Pair } from '@/interface.js' import type { Datastore } from 'interface-datastore' import { OperationManager } from '@/operation-manager.js' +import { CustomEvent } from '@libp2p/interface' export default async (context: Pick, { events, pinManager, helia }: Components, datastore: Datastore): Promise diff --git a/packages/daemon/src/modules/revisions/setup.ts b/packages/daemon/src/modules/revisions/setup.ts index cf94b637..dd232eca 100644 --- a/packages/daemon/src/modules/revisions/setup.ts +++ b/packages/daemon/src/modules/revisions/setup.ts @@ -5,6 +5,7 @@ import { pathToKey } from './utils.js' import { type Context, logger } from './index.js' import type { Components } from '@/common/interface.js' import type { CID } from 'multiformats/cid' +import { CustomEvent } from '@libp2p/interface' export default async ({ groups, welo, events }: Components): Promise => { const getRevisions = (group: CID): Revisions | null => { From eb8e9cc6e1894928985b8a3508808d5fdcc0f2fb Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 12:20:50 +1300 Subject: [PATCH 32/62] Disable daemon tests. --- packages/daemon/package.json | 3 +-- packages/daemon/tsconfig.json | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 5bf03e1d..4551c06b 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -19,8 +19,7 @@ "lint": "aegir lint", "dep-check": "aegir dep-check", "prepublishOnly": "npm run build", - "build": "aegir build && tsc-alias", - "test": "aegir test -t node -f './dist/test/**/*.spec.js'" + "build": "aegir build && tsc-alias" }, "author": "Saul Boyd", "license": "GPL-3.0-or-later", diff --git a/packages/daemon/tsconfig.json b/packages/daemon/tsconfig.json index f3691caf..a94b36f8 100644 --- a/packages/daemon/tsconfig.json +++ b/packages/daemon/tsconfig.json @@ -13,8 +13,7 @@ } }, "include": [ - "src", - "test" + "src" ], "references": [ { From 8ff4e69df5e5861b31a7c116ca73c86d4b46c3bd Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 12:25:39 +1300 Subject: [PATCH 33/62] Nuke the daemon tests. --- packages/daemon/test/modules/argv.spec.ts | 33 - packages/daemon/test/modules/base.spec.ts | 147 ----- packages/daemon/test/modules/config.spec.ts | 54 -- .../daemon/test/modules/downloader.spec.ts | 236 ------- .../daemon/test/modules/filesystem.spec.ts | 618 ------------------ packages/daemon/test/modules/groups.spec.ts | 311 --------- packages/daemon/test/modules/mock-argv.ts | 12 - packages/daemon/test/modules/mock-base.ts | 20 - packages/daemon/test/modules/mock-config.ts | 12 - packages/daemon/test/modules/network.spec.ts | 313 --------- .../daemon/test/modules/revisions.spec.ts | 460 ------------- packages/daemon/test/modules/rpc.spec.ts | 61 -- packages/daemon/test/modules/sigint.spec.ts | 15 - packages/daemon/test/modules/tick.spec.ts | 69 -- packages/daemon/test/utils.spec.ts | 114 ---- packages/daemon/test/utils/blocks.ts | 46 -- packages/daemon/test/utils/create-group.ts | 17 - packages/daemon/test/utils/dag.ts | 44 -- packages/daemon/test/utils/generate-key.ts | 14 - packages/daemon/test/utils/paths.ts | 6 - packages/daemon/tsconfig.json | 3 +- 21 files changed, 2 insertions(+), 2603 deletions(-) delete mode 100644 packages/daemon/test/modules/argv.spec.ts delete mode 100644 packages/daemon/test/modules/base.spec.ts delete mode 100644 packages/daemon/test/modules/config.spec.ts delete mode 100644 packages/daemon/test/modules/downloader.spec.ts delete mode 100644 packages/daemon/test/modules/filesystem.spec.ts delete mode 100644 packages/daemon/test/modules/groups.spec.ts delete mode 100644 packages/daemon/test/modules/mock-argv.ts delete mode 100644 packages/daemon/test/modules/mock-base.ts delete mode 100644 packages/daemon/test/modules/mock-config.ts delete mode 100644 packages/daemon/test/modules/network.spec.ts delete mode 100644 packages/daemon/test/modules/revisions.spec.ts delete mode 100644 packages/daemon/test/modules/rpc.spec.ts delete mode 100644 packages/daemon/test/modules/sigint.spec.ts delete mode 100644 packages/daemon/test/modules/tick.spec.ts delete mode 100644 packages/daemon/test/utils.spec.ts delete mode 100644 packages/daemon/test/utils/blocks.ts delete mode 100644 packages/daemon/test/utils/create-group.ts delete mode 100644 packages/daemon/test/utils/dag.ts delete mode 100644 packages/daemon/test/utils/generate-key.ts delete mode 100644 packages/daemon/test/utils/paths.ts diff --git a/packages/daemon/test/modules/argv.spec.ts b/packages/daemon/test/modules/argv.spec.ts deleted file mode 100644 index 483f2fa8..00000000 --- a/packages/daemon/test/modules/argv.spec.ts +++ /dev/null @@ -1,33 +0,0 @@ -import assert from 'assert/strict' -import Path from 'path' -import argv from '../../src/modules/argv/index.js' -import { projectPath } from '@/utils.js' - -describe('argv', () => { - it('returns defaults for every argv parameter', async () => { - const m = await argv() - - assert.equal(m.key, Path.join(projectPath, 'config/key.json')) - assert.equal(m.config, Path.join(projectPath, 'config/config.json')) - assert.equal(m.socket, '/tmp/server.socket') - }) - - it('returns the value for every argv parameter', async () => { - const key = '/key.json' - const config = '/config.json' - const socket = '/socket' - - process.argv.push('--key') - process.argv.push(key) - process.argv.push('--config') - process.argv.push(config) - process.argv.push('--socket') - process.argv.push(socket) - - const m = await argv() - - assert.equal(m.key, key) - assert.equal(m.config, config) - assert.equal(m.socket, socket) - }) -}) diff --git a/packages/daemon/test/modules/base.spec.ts b/packages/daemon/test/modules/base.spec.ts deleted file mode 100644 index 81f9c180..00000000 --- a/packages/daemon/test/modules/base.spec.ts +++ /dev/null @@ -1,147 +0,0 @@ -import assert from 'assert/strict' -import fs from 'fs/promises' -import Path from 'path' -import { MemoryBlockstore } from 'blockstore-core' -import { FsBlockstore } from 'blockstore-fs' -import { MemoryDatastore } from 'datastore-core' -import { FsDatastore } from 'datastore-fs' -import { Key } from 'interface-datastore' -import { CID } from 'multiformats/cid' -import { fromString as uint8ArrayFromString } from 'uint8arrays' -import base from '../../src/modules/base/index.js' -import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockConfig from './mock-config.js' -import type { Provides as Argv } from '../../src/modules/argv/index.js' -import { extendDatastore } from '@/utils.js' - -const parseStr = (data: string): Uint8Array => uint8ArrayFromString(data, 'base64') -const testPath = mkTestPath('base') - -describe('base', () => { - let argv: Argv - - before(async () => { - argv = mockArgv(testPath) - - await fs.mkdir(Path.join(argv.key, '..'), { recursive: true }) - await fs.mkdir(testPath, { recursive: true }) - - await fs.writeFile(argv.key, JSON.stringify({ - key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' - })) - }) - - after(async () => { - await fs.rm(testPath, { recursive: true }) - }) - - it('returns the key manager', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) - - assert.deepEqual( - new Uint8Array(m.keyManager.aesKey), - parseStr('knUGn6uUeQcoxfM1qAtg3F/Njm4bp+GcZK257NZ5AtE') - ) - - assert.deepEqual( - new Uint8Array(m.keyManager.hmacKey), - parseStr('KZkJfNz3bRrn6XHvWYtD4+dXvmhdT4TBhBIkWn8y3jY') - ) - - assert.deepEqual( - (await m.keyManager.getWeloIdentity()).id, - parseStr('CAISIQPSvjmKINqJY5SA/3c+kadFmIsHeTXtTJYlrooZ53DTUg') - ) - - assert.deepEqual( - (await m.keyManager.getPeerId()).toBytes(), - parseStr('ACUIAhIhA5m2/DfXxqi0i+fyYixRaWGirDEVemxUEv8WMZPwFPZB') - ) - - assert.deepEqual( - m.keyManager.getPskKey(), - parseStr('L2tleS9zd2FybS9wc2svMS4wLjAvCi9iYXNlMTYvCjU2ZDNjMTgyODJmMWYxYjFiM2UwNGU0MGRkNWQ4YmY0NGNhZmE4YmM5YzliYzdjNTc3MTZhNzc2NmZhMmM1NTA') - ) - }) - - it('uses memory blockstore when memory is specified', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) - - assert(m.blockstore instanceof MemoryBlockstore) - }) - - it('uses memory datastore when memory is specified', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) - - assert(m.datastore instanceof MemoryDatastore) - }) - - it('uses fs blockstore when a path is specified', async () => { - const blockstorePath = Path.join(testPath, 'blockstore') - const testData = uint8ArrayFromString('test') - - const m = await base({ - config: mockConfig({ storage: testPath }), - argv - }) - - assert(m.blockstore instanceof FsBlockstore) - - await m.blockstore.put(CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ'), testData) - - const out = await fs.readdir(blockstorePath, { recursive: true }) - - assert.deepEqual(out, [ - '7I', - '7I/BCIQLASSX2QHMUE4IBHYTTJ3LCICEGM6DOQBZDSN7DTU5RYQ2PEQQX7I.data' - ]) - - const blockData = await fs.readFile(Path.join(blockstorePath, '7I/BCIQLASSX2QHMUE4IBHYTTJ3LCICEGM6DOQBZDSN7DTU5RYQ2PEQQX7I.data')) - - assert.deepEqual(new Uint8Array(blockData), testData) - }) - - it('uses fs datastore when a path is specified', async () => { - const datastorePath = Path.join(testPath, 'datastore') - - const m = await base({ - config: mockConfig({ storage: testPath }), - argv - }) - - assert(m.datastore instanceof FsDatastore) - - await m.datastore.put(new Key('key'), uint8ArrayFromString('value')) - - const subDatastore = extendDatastore(extendDatastore(m.datastore, 'a'), 'b/c') - await subDatastore.put(new Key('d/e'), uint8ArrayFromString('test')) - - const out = await fs.readdir(datastorePath, { recursive: true }) - - assert.deepEqual(out, [ - 'a', - 'key.data', - 'a/b', - 'a/b/c', - 'a/b/c/d', - 'a/b/c/d/e.data' - ]) - - const data1 = await fs.readFile(Path.join(datastorePath, 'key.data')) - const data2 = await fs.readFile(Path.join(datastorePath, 'a/b/c/d/e.data')) - - assert.deepEqual(new Uint8Array(data1), uint8ArrayFromString('value')) - assert.deepEqual(new Uint8Array(data2), uint8ArrayFromString('test')) - }) -}) diff --git a/packages/daemon/test/modules/config.spec.ts b/packages/daemon/test/modules/config.spec.ts deleted file mode 100644 index 26c93b57..00000000 --- a/packages/daemon/test/modules/config.spec.ts +++ /dev/null @@ -1,54 +0,0 @@ -import assert from 'assert/strict' -import fs from 'fs/promises' -import Path from 'path' -import { z } from 'zod' -import config from '../../src/modules/config/index.js' -import { mkTestPath } from '../utils/paths.js' - -const testPath = mkTestPath('config') -const configPath = Path.join(testPath, 'config.json') - -const configData = { - private: true, - tickInterval: 1000, - storage: ':memory:', - addresses: [ - '/ip4/0.0.0.0/tcp/0' - ], - bootstrap: ['/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'] -} - -before(async () => { - await fs.mkdir(testPath, { recursive: true }) - await fs.writeFile(configPath, JSON.stringify(configData)) -}) - -after(async () => { - await fs.rm(testPath, { recursive: true }) -}) - -describe('config', () => { - it('returns parsed config from the file', async () => { - const m = await config({ - argv: { config: configPath, key: '', socket: '' } - }) - - assert.deepEqual(m.config, configData) - }) - - it('gets config from schema', async () => { - const m = await config({ - argv: { config: configPath, key: '', socket: '' } - }) - - assert.deepEqual( - m.get(z.object({ bootstrap: z.array(z.string()) })), - { bootstrap: configData.bootstrap } - ) - - assert.deepEqual( - m.get(z.object({ tickInterval: z.number(), private: z.boolean() })), - { tickInterval: configData.tickInterval, private: configData.private } - ) - }) -}) diff --git a/packages/daemon/test/modules/downloader.spec.ts b/packages/daemon/test/modules/downloader.spec.ts deleted file mode 100644 index 5c459870..00000000 --- a/packages/daemon/test/modules/downloader.spec.ts +++ /dev/null @@ -1,236 +0,0 @@ -import assert from 'assert/strict' -import fs from 'fs/promises' -import Path from 'path' -import { KeyManager } from '@organicdesign/db-key-manager' -import { createNetClient } from '@organicdesign/net-rpc' -import { MemoryBlockstore } from 'blockstore-core' -import { CID } from 'multiformats/cid' -import createDownloader from '../../src/modules/downloader/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' -import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { - Requires as DownloaderComponents, - Provides as DownloaderProvides -} from '../../src/modules/downloader/index.js' - -describe('downloader', () => { - const testPath = mkTestPath('groups') - - const create = async (name?: string): Promise & { - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - downloader: DownloaderProvides - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - downloader - } - } - - before(async () => { - await fs.mkdir(testPath, { recursive: true }) - }) - - after(async () => { - await fs.rm(testPath, { recursive: true }) - }) - - it('has 20 slots by default', async () => { - const { downloader: m, sigint } = await create() - - assert.equal(m.config.slots, 20) - - await sigint.interupt() - }) - - it('rpc - set priority updates local priority', async () => { - const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' - const path = '/test.txt' - const priority = 50 - const { downloader: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const key = Path.join('/', group, path) - - await m.pinManager.put(key, { priority: 1, cid: CID.parse(group) }) - - const response = await client.rpc.request('set-priority', { group, path, priority }) - - assert.equal(response, null) - - const pinData = await m.pinManager.get(key) - - assert(pinData != null) - assert.equal(pinData.priority, priority) - - client.close() - await sigint.interupt() - }) - - it('rpc - get speed', async () => { - const { downloader: m, network, sigint, argv } = await create() - const blockstore = new MemoryBlockstore() - const dag = await createDag({ blockstore }, 2, 2) - const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' - const path = '/test.txt' - const range = 500 - const client = createNetClient(argv.socket) - const key = Path.join('/', group, path) - - await m.pinManager.put(key, { priority: 1, cid: dag[0] }) - - const speed1 = await client.rpc.request('get-speeds', { - cids: [dag[0].toString()], - range - }) - - assert.deepEqual(speed1, [{ cid: dag[0].toString(), speed: 0 }]) - - const value = await blockstore.get(dag[0]) - - await network.helia.blockstore.put(dag[0], value) - await new Promise(resolve => setTimeout(resolve, range / 2)) - - const speed2 = await client.rpc.request('get-speeds', { - cids: [dag[0].toString()], - range - }) - - assert.deepEqual(speed2, [{ cid: dag[0].toString(), speed: value.length / range }]) - - await new Promise(resolve => setTimeout(resolve, range / 2)) - - const values = await Promise.all([ - blockstore.get(dag[1]), - blockstore.get(dag[4]) - ]) - - await Promise.all([ - network.helia.blockstore.put(dag[1], values[0]), - network.helia.blockstore.put(dag[4], values[1]) - ]) - - await new Promise(resolve => setTimeout(resolve, range / 2)) - - const speed3 = await client.rpc.request('get-speeds', { - cids: [dag[0].toString()], - range - }) - - assert.deepEqual(speed3, [{ - cid: dag[0].toString(), - speed: values.reduce((a, c) => c.length + a, 0) / range - }]) - - client.close() - await sigint.interupt() - }) - - it('rpc - get status', async () => { - const { downloader: m, network, sigint, argv } = await create() - const blockstore = new MemoryBlockstore() - const dag = await createDag({ blockstore }, 2, 2) - const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' - const path = '/test.txt' - const client = createNetClient(argv.socket) - const key = Path.join('/', group, path) - - const status1 = await client.rpc.request('get-status', { - cids: [dag[0].toString()] - }) - - assert.deepEqual(status1, [{ - cid: dag[0].toString(), - blocks: 0, - size: 0, - state: 'NOTFOUND' - }]) - - await m.pinManager.put(key, { priority: 1, cid: dag[0] }) - - const status2 = await client.rpc.request('get-status', { - cids: [dag[0].toString()] - }) - - assert.deepEqual(status2, [{ - cid: dag[0].toString(), - blocks: 0, - size: 0, - state: 'DOWNLOADING' - }]) - - const value = await blockstore.get(dag[0]) - - await network.helia.blockstore.put(dag[0], value) - await new Promise(resolve => setTimeout(resolve, 100)) - - const status3 = await client.rpc.request('get-status', { - cids: [dag[0].toString()] - }) - - assert.deepEqual(status3, [{ - cid: dag[0].toString(), - blocks: 1, - size: value.length, - state: 'DOWNLOADING' - }]) - - const values = await Promise.all(dag.map(async cid => { - const value = await blockstore.get(cid) - - await network.helia.blockstore.put(cid, value) - - return value.length - })) - - await new Promise(resolve => setTimeout(resolve, 500)) - - const status4 = await client.rpc.request('get-status', { - cids: [dag[0].toString()] - }) - - assert.deepEqual(status4, [{ - cid: dag[0].toString(), - blocks: dag.length, - size: values.reduce((a, c) => a + c, 0), - state: 'COMPLETED' - }]) - - client.close() - await sigint.interupt() - }) -}) diff --git a/packages/daemon/test/modules/filesystem.spec.ts b/packages/daemon/test/modules/filesystem.spec.ts deleted file mode 100644 index 2c1d8896..00000000 --- a/packages/daemon/test/modules/filesystem.spec.ts +++ /dev/null @@ -1,618 +0,0 @@ -import assert from 'assert/strict' -import fs from 'fs/promises' -import Path from 'path' -import { unixfs } from '@helia/unixfs' -import { KeyManager } from '@organicdesign/db-key-manager' -import * as testData from '@organicdesign/db-test-utils' -import { importer } from '@organicdesign/db-utils' -import { createNetClient } from '@organicdesign/net-rpc' -import { MemoryBlockstore } from 'blockstore-core' -import all from 'it-all' -import { CID } from 'multiformats/cid' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createDownloader from '../../src/modules/downloader/index.js' -import createFilesystem from '../../src/modules/filesystem/index.js' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import createTick from '../../src/modules/tick/index.js' -import { createGroup } from '../utils/create-group.js' -import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' -import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { Provides as FSProvides } from '../../src/modules/filesystem/index.js' - -describe('filesystem', () => { - const testPath = mkTestPath('filesystem') - - const create = async (name?: string): Promise<{ - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: Awaited> - filesystem: FSProvides - sigint: Awaited> - tick: Awaited> - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) - - const tick = await createTick({ config, sigint }) - - const filesystem = await createFilesystem({ - sigint, - base, - rpc, - network, - groups, - downloader, - tick, - config - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups, - filesystem, - tick - } - } - - before(async () => { - await fs.mkdir(testPath, { recursive: true }) - }) - - after(async () => { - await fs.rm(testPath, { recursive: true }) - }) - - it('uses all as the default revision strategy', async () => { - const { filesystem: m, sigint } = await create() - - assert.equal(m.config.defaultRevisionStrategy, 'all') - - await sigint.interupt() - }) - - it('returns null when wraping a group that doesn\'t exist', async () => { - const { filesystem: m, sigint } = await create() - const fs = m.getFileSystem(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) - - assert.equal(fs, null) - - await sigint.interupt() - }) - - it('wraps a group in a filesystem', async () => { - const { filesystem: m, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - - assert.notEqual(fs, null) - - await sigint.interupt() - }) - - it('uploads show in filesystem', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) - const path = '/test' - - assert(fs != null) - - const before = Date.now() - - await m.uploads.add('put', [group.bytes, path, { - cid: dag[0].bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - const entry = await fs.get(path) - const values = await Promise.all(dag.map(async d => network.helia.blockstore.get(d))) - const size = values.reduce((a, c) => a + c.length, 0) - - assert(entry != null) - assert.deepEqual(entry.author, groups.welo.identity.id) - assert.equal(entry.blocks, dag.length) - assert.deepEqual(entry.cid, dag[0]) - assert.equal(entry.encrypted, false) - assert.equal(entry.priority, 1) - assert.equal(entry.revisionStrategy, 'all') - assert.equal(entry.sequence, 0) - assert.equal(entry.size, size) - assert(entry.timestamp >= before) - assert(entry.timestamp <= Date.now()) - - await sigint.interupt() - }) - - it('emits the file:added event when the FS was written to', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const dag = await createDag(network.helia, 2, 2) - const path = '/test' - - const promise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, 100) - - m.events.addEventListener('file:added', () => { resolve() }, { once: true }) - }) - - await m.uploads.add('put', [group.bytes, path, { - cid: dag[0].bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - await promise - - await sigint.interupt() - }) - - it('local settings change FS output', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) - const path = '/test' - - assert(fs != null) - - await m.uploads.add('put', [group.bytes, path, { - cid: dag[0].bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - await m.localSettings.set(group, path, { - priority: 100 - }) - - const entry = await fs.get(path) - - assert(entry != null) - assert.equal(entry.priority, 100) - - await sigint.interupt() - }) - - it('rpc - delete (file)', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) - const path = '/test' - - assert(fs != null) - - await m.uploads.add('put', [group.bytes, path, { - cid: dag[0].bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - const response = await client.rpc.request('delete', { group: group.toString(), path }) - - assert.deepEqual(response, [{ path, cid: dag[0].toString() }]) - - const entry = await fs.get(path) - - assert.equal(entry, null) - - client.close() - await sigint.interupt() - }) - - it('rpc - delete (directory)', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) - const rootPath = '/test' - const paths = [`${rootPath}/file1`, `${rootPath}/file2`, `${rootPath}/sub/file3`] - - assert(fs != null) - - await Promise.all(paths.map(async path => - m.uploads.add('put', [group.bytes, path, { - cid: dag[0].bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - )) - - const response = await client.rpc.request('delete', { group: group.toString(), path: rootPath }) - - assert.deepEqual(response, paths.map(path => ({ path, cid: dag[0].toString() }))) - - const entries = await all(fs.getDir(rootPath)) - - assert.deepEqual(entries, []) - - client.close() - await sigint.interupt() - }) - - it('rpc - edit priority', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const path = '/test' - const priority = 50 - - assert(fs != null) - - const response = await client.rpc.request('edit', { group: group.toString(), path, priority }) - - assert.equal(response, null) - - const localSettings = await m.localSettings.get(group, path) - - assert.equal(localSettings.priority, priority) - - client.close() - await sigint.interupt() - }) - - it('rpc - export (file)', async () => { - const { filesystem: m, network, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const rootPath = '/test' - const outPath = Path.join(testPath, 'export-file') - - assert(fs != null) - - await Promise.all(testData.data.map(async data => { - const result = await all(importer(network.helia.blockstore, data.path)) - - await m.uploads.add('put', [group.bytes, data.generatePath(rootPath), { - cid: result[0].cid.bytes, - encrypted: false, - revisionStrategy: 'all', - priority: 1 - }]) - })) - - for (const data of testData.data) { - const exportPath = data.generatePath(outPath) - - const response = await client.rpc.request('export', { - group: group.toString(), - path: data.generatePath(rootPath), - outPath: exportPath - }) - - const valid = await data.validate(exportPath) - - assert.equal(response, null) - assert.equal(valid, true) - } - - client.close() - await sigint.interupt() - }) - - it('rpc - export (directory)', async () => { - const { filesystem: m, network, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const rootPath = '/test' - const outPath = Path.join(testPath, 'export-directory') - - assert(fs != null) - - await Promise.all(testData.data.map(async data => { - const result = await all(importer(network.helia.blockstore, data.path)) - - await m.uploads.add('put', [group.bytes, data.generatePath(rootPath), { - cid: result[0].cid.bytes, - encrypted: false, - revisionStrategy: 'all', - priority: 1 - }]) - })) - - const response = await client.rpc.request('export', { - group: group.toString(), - path: rootPath, - outPath - }) - - assert.equal(response, null) - - for (const data of testData.data) { - const exportPath = data.generatePath(outPath) - const valid = await data.validate(exportPath) - - assert.equal(valid, true) - } - - client.close() - await sigint.interupt() - }) - - it('rpc - import (file)', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const rootPath = '/test' - - assert(fs != null) - - await Promise.all(testData.data.map(async data => { - const virtualPath = data.generatePath(rootPath) - - const response = await client.rpc.request('import', { - group: group.toString(), - path: virtualPath, - inPath: data.path - }) - - assert.deepEqual(response, [{ - path: virtualPath, - inPath: data.path, - cid: data.cid.toString() - }]) - - const result = await fs.get(virtualPath) - - assert(result != null) - assert.deepEqual(result.cid, data.cid) - })) - - client.close() - await sigint.interupt() - }) - - it('rpc - import (directory)', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const rootPath = '/test' - - assert(fs != null) - - const response = await client.rpc.request('import', { - group: group.toString(), - path: rootPath, - inPath: testData.root - }) - - assert(Array.isArray(response)) - assert.equal(response.length, testData.data.length) - - for (const data of response) { - const dataFile = testData.getDataFile(data.inPath) - - assert(dataFile != null) - assert.equal(data.path, dataFile.generatePath(rootPath)) - assert.equal(data.cid, dataFile.cid.toString()) - } - - const fsResult = await all(fs.getDir(rootPath)) - - assert(Array.isArray(fsResult)) - assert.equal(fsResult.length, testData.data.length) - - for (const dataFile of testData.data) { - const virtualPath = dataFile.generatePath(rootPath) - const r = fsResult.find(r => r.key === virtualPath) - - assert(r != null) - assert.deepEqual(r.value.cid, dataFile.cid) - } - - client.close() - await sigint.interupt() - }) - - it('rpc - list', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) - const rootPath = '/test' - const paths = [`${rootPath}/file1`, `${rootPath}/file2`, `${rootPath}/sub/file3`] - - assert(fs != null) - - const before = Date.now() - - await Promise.all(paths.map(async path => - m.uploads.add('put', [group.bytes, path, { - cid: dag[0].bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - )) - - const response = await client.rpc.request('list', { group: group.toString(), path: '/' }) - - assert(Array.isArray(response)) - - const values = await Promise.all(dag.map(async d => network.helia.blockstore.get(d))) - const size = values.reduce((a, c) => a + c.length, 0) - - for (const entry of response) { - assert(entry != null) - assert.equal(entry.author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) - assert.equal(entry.blocks, dag.length) - assert.deepEqual(entry.cid, dag[0].toString()) - assert.equal(entry.encrypted, false) - assert.equal(entry.priority, 1) - assert.equal(entry.revisionStrategy, 'all') - assert.equal(entry.size, size) - assert(entry.timestamp >= before) - assert(entry.timestamp <= Date.now()) - } - - client.close() - await sigint.interupt() - }) - - it('rpc - read', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const ufs = unixfs(network.helia) - const path = '/test' - const data = 'test-data' - - const cid = await ufs.addBytes(uint8ArrayFromString(data)) - - await m.uploads.add('put', [group.bytes, path, { - cid: cid.bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - const read1 = await client.rpc.request('read', { group: group.toString(), path }) - - assert.deepEqual(read1, data) - - const read2 = await client.rpc.request('read', { group: group.toString(), path, position: 1 }) - - assert.deepEqual(read2, data.slice(1)) - - const read3 = await client.rpc.request('read', { group: group.toString(), path, length: 3 }) - - assert.deepEqual(read3, data.slice(0, 3)) - - const read4 = await client.rpc.request('read', { group: group.toString(), path, position: 1, length: 3 }) - - assert.deepEqual(read4, data.slice(1, 3 + 1)) - - client.close() - await sigint.interupt() - }) - - it('rpc - write', async () => { - const { filesystem: m, base, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const ufs = unixfs({ blockstore: new MemoryBlockstore() }) - const path = '/test' - const data = 'test-data' - const cid = await ufs.addBytes(uint8ArrayFromString(data)) - - assert(fs != null) - - const before = Date.now() - - const write1 = await client.rpc.request('write', { group: group.toString(), path, data }) - - assert.equal(write1, data.length) - - const entry1 = await fs.get(path) - - assert(entry1 != null) - assert.deepEqual(entry1.author, groups.welo.identity.id) - assert.equal(entry1.blocks, 1) - assert.deepEqual(entry1.cid, cid) - assert.equal(entry1.encrypted, false) - assert.equal(entry1.priority, 100) - assert.equal(entry1.revisionStrategy, 'all') - assert.equal(entry1.size, data.length) - assert(entry1.timestamp >= before) - assert(entry1.timestamp <= Date.now()) - - const newData2 = 'your-data-long' - const write2 = await client.rpc.request('write', { group: group.toString(), path, data: newData2 }) - - assert.equal(write2, newData2.length) - - const entry2 = await fs.get(path) - - assert(entry2 != null) - - const value2 = await base.blockstore.get(entry2.cid) - - assert.deepEqual(value2, uint8ArrayFromString(newData2)) - - const newData3 = 'test' - const write3 = await client.rpc.request('write', { group: group.toString(), path, data: newData3, length: newData3.length }) - - assert.equal(write3, newData3.length) - - const entry3 = await fs.get(path) - - assert(entry3 != null) - - const value3 = await base.blockstore.get(entry3.cid) - - assert.deepEqual(value3, uint8ArrayFromString('test-data-long')) - - const newData4 = 'long' - const write4 = await client.rpc.request('write', { group: group.toString(), path, data: newData4, length: newData4.length, position: 5 }) - - assert.equal(write4, newData4.length) - - const entry4 = await fs.get(path) - - assert(entry4 != null) - - const value4 = await base.blockstore.get(entry4.cid) - - assert.deepEqual(value4, uint8ArrayFromString('test-long-long')) - - client.close() - await sigint.interupt() - }) -}) diff --git a/packages/daemon/test/modules/groups.spec.ts b/packages/daemon/test/modules/groups.spec.ts deleted file mode 100644 index f94d84f9..00000000 --- a/packages/daemon/test/modules/groups.spec.ts +++ /dev/null @@ -1,311 +0,0 @@ -import assert from 'assert/strict' -import fs from 'fs/promises' -import Path from 'path' -import { KeyManager, parseKeyData } from '@organicdesign/db-key-manager' -import { createNetClient } from '@organicdesign/net-rpc' -import * as cborg from 'cborg' -import all from 'it-all' -import { CID } from 'multiformats/cid' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { - Requires as GroupsComponents, - Provides as GroupsProvides -} from '../../src/modules/groups/index.js' - -describe('groups', () => { - const testPath = mkTestPath('groups') - - const mkGroup = async (m: GroupsProvides, name: string, peers: Uint8Array[] = []): Promise => { - const manifest = await m.welo.determine({ - name, - meta: { type: 'group' }, - access: { - protocol: '/hldb/access/static', - config: { write: [m.welo.identity.id, ...peers] } - } - }) - - await m.groups.add(manifest) - - return manifest.address.cid - } - - const create = async (name?: string): Promise & { - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: GroupsProvides - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - - const keyManager = name == null - ? undefined - : new KeyManager(parseKeyData({ - key: 'DpGbLiAX4wK4HHtG3DQb8cA6FG2ibv93X4ooZJ5LmMJJ-12FmenN8dbWysuYnzEHzmEF1hod4RGK8NfKFu1SEZ7XM', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n023330a98e30315e2233d4a31a6dc65d741a89f7ce6248e7de40c73995d23157' - })) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups - } - } - - before(async () => { - await fs.mkdir(testPath, { recursive: true }) - }) - - after(async () => { - await fs.rm(testPath, { recursive: true }) - }) - - it('creates a group', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - - assert(group) - - await sigint.interupt() - }) - - it('tracker puts and checks a group\'s content', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) - - if (database == null) { - throw new Error('group creation failed') - } - - const tracker = m.getTracker(database) - - const key = 'test' - const put = database.store.creators.put(key, 'my-data') - - await database.replica.write(put) - - assert.equal(await tracker.validate(key, put), false) - - await tracker.put(key, put) - - assert.equal(await tracker.validate(key, put), true) - - await sigint.interupt() - }) - - it('tracker processes a group\'s content', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) - - if (database == null) { - throw new Error('group creation failed') - } - - const tracker = m.getTracker(database) - const key = '/test' - const value = 'my-data' - const put = database.store.creators.put(key, value) - - let entries = await all(tracker.process()) - assert.deepEqual(entries, []) - - await database.replica.write(put) - entries = await all(tracker.process()) - assert.deepEqual(entries, [{ key, value: cborg.encode(value) }]) - - entries = await all(tracker.process()) - assert.deepEqual(entries, []) - - await sigint.interupt() - }) - - it('tracker is scope limited', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) - - if (database == null) { - throw new Error('group creation failed') - } - - const tracker = m.getTracker(database) - const key = '/test' - const value = 'my-data' - const put = database.store.creators.put(key, value) - - await database.replica.write(put) - - let entries = await all(tracker.process('/another-key')) - assert.deepEqual(entries, []) - - entries = await all(tracker.process(key)) - assert.deepEqual(entries, [{ key, value: cborg.encode(value) }]) - - await sigint.interupt() - }) - - it('uses the identity from base in welo', async () => { - const { groups: m, sigint, base } = await create() - - assert.deepEqual(m.welo.identity, await base.keyManager.getWeloIdentity()) - - await sigint.interupt() - }) - - it('rpc - id returns the base58btc formatted welo id', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) - - const id = await client.rpc.request('id', {}) - - assert.equal(uint8ArrayToString(m.welo.identity.id, 'base58btc'), id) - - client.close() - await sigint.interupt() - }) - - it('rpc - create groups creates a group without other peers', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const name = 'test' - - const cid = await client.rpc.request('create-group', { name, peers: [] }) - const group = CID.parse(cid) - const database = m.groups.get(group) - - assert(database != null) - assert.equal(database.manifest.name, name) - assert.deepEqual(database.manifest.access.config?.write, [m.welo.identity.id]) - - client.close() - await sigint.interupt() - }) - - it('rpc - create groups creates a group with other peers', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const name = 'test' - const otherPeer = 'GZsJqUjmbVqZCUMbJoe5ye4xfdKZVPVwBoFFQiyCZYesq6Us5b' - - const cid = await client.rpc.request('create-group', { name, peers: [otherPeer] }) - const group = CID.parse(cid) - const database = m.groups.get(group) - - assert(database != null) - assert.equal(database.manifest.name, name) - assert.deepEqual(database.manifest.access.config?.write, [ - m.welo.identity.id, - uint8ArrayFromString(otherPeer, 'base58btc') - ]) - - client.close() - await sigint.interupt() - }) - - it('rpc - joins an external group', async () => { - const components = await Promise.all([create(), create('server-join-group')]) - const client = createNetClient(components[0].argv.socket) - const name = 'test' - const group = await mkGroup(components[1].groups, name) - - await components[0].network.libp2p.dial(components[1].network.libp2p.getMultiaddrs()) - const res = await client.rpc.request('join-group', { group: group.toString() }) - - assert.equal(res, null) - - const database = components[0].groups.groups.get(group) - - assert(database) - assert.equal(database.manifest.name, name) - - client.close() - await Promise.all(components.map(async c => c.sigint.interupt())) - }) - - it('rpc - list groups', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const name = 'test' - - let groups = await client.rpc.request('list-groups', {}) - - assert.deepEqual(groups, []) - - const group = await mkGroup(m, name) - - groups = await client.rpc.request('list-groups', {}) - - assert.deepEqual(groups, [{ group: group.toString(), name }]) - - client.close() - await sigint.interupt() - }) - - // This fails it github too - seems to think the `server-sync-groups` socket is in use? - it.skip('rpc - sync groups', async () => { - const components = await Promise.all([create(), create('server-sync-groups')]) - const client = createNetClient(components[0].argv.socket) - const key = '/test' - const value = 'test-value' - - const groups = await client.rpc.request('list-groups', {}) - - assert.deepEqual(groups, []) - - const group = await mkGroup(components[1].groups, 'test') - const database = components[1].groups.groups.get(group) - - if (database == null) { - throw new Error('database creation failed') - } - - const put = database.store.creators.put(key, value) - - await database.replica.write(put) - - await components[0].network.libp2p.dial(components[1].network.libp2p.getMultiaddrs()) - await client.rpc.request('join-group', { group: group.toString() }) - await client.rpc.request('sync', {}) - - const index = await database.store.latest() - const result = await database.store.selectors.get(index)(key) - - assert.deepEqual(result, value) - - client.close() - await Promise.all(components.map(async c => c.sigint.interupt())) - }) -}) diff --git a/packages/daemon/test/modules/mock-argv.ts b/packages/daemon/test/modules/mock-argv.ts deleted file mode 100644 index c6da9904..00000000 --- a/packages/daemon/test/modules/mock-argv.ts +++ /dev/null @@ -1,12 +0,0 @@ -import Path from 'path' -import type { Provides } from '../../src/modules/argv/index.js' - -const provs = (path: string): Provides => { - return { - key: Path.join(path, 'key.json'), - config: Path.join(path, 'config.json'), - socket: Path.join(path, 'server.socket') - } -} - -export default provs diff --git a/packages/daemon/test/modules/mock-base.ts b/packages/daemon/test/modules/mock-base.ts deleted file mode 100644 index a139b417..00000000 --- a/packages/daemon/test/modules/mock-base.ts +++ /dev/null @@ -1,20 +0,0 @@ -import Path from 'path' -import { KeyManager, parseKeyData } from '@organicdesign/db-key-manager' -import { MemoryBlockstore } from 'blockstore-core' -import { FsBlockstore } from 'blockstore-fs' -import { MemoryDatastore } from 'datastore-core' -import { FsDatastore } from 'datastore-fs' -import type { Provides } from '../../src/modules/base/index.js' - -const provs = (config?: { path?: string, keyManager?: KeyManager }): Provides => { - return { - keyManager: config?.keyManager ?? new KeyManager(parseKeyData({ - key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' - })), - datastore: config?.path == null ? new MemoryDatastore() : new FsDatastore(Path.join(config?.path, 'datastore')), - blockstore: config?.path == null ? new MemoryBlockstore() : new FsBlockstore(Path.join(config?.path, 'blockstore')) - } -} - -export default provs diff --git a/packages/daemon/test/modules/mock-config.ts b/packages/daemon/test/modules/mock-config.ts deleted file mode 100644 index 45bf3666..00000000 --- a/packages/daemon/test/modules/mock-config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { type z } from 'zod' -import type { Provides } from '../../src/modules/config/index.js' - -const provs = (config: Record): Provides => ({ - config, - - get (schema: T): z.infer { - return schema.parse(config) - } -}) - -export default provs diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts deleted file mode 100644 index 5b83c21d..00000000 --- a/packages/daemon/test/modules/network.spec.ts +++ /dev/null @@ -1,313 +0,0 @@ -import assert from 'assert/strict' -import fs from 'fs/promises' -import { unixfs } from '@helia/unixfs' -import { createNetClient } from '@organicdesign/net-rpc' -import { createHelia } from 'helia' -import { Key } from 'interface-datastore' -import all from 'it-all' -import { CID } from 'multiformats/cid' -import network from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { Requires as NetworkComponents } from '../../src/modules/network/index.js' -import type { Libp2p } from '@libp2p/interface' -import createLibp2p from '@/modules/network/libp2p.js' - -describe('network', () => { - const testPath = mkTestPath('network') - let components: NetworkComponents & { argv: ReturnType } - - before(async () => { - await fs.mkdir(testPath, { recursive: true }) - - const sigint = await createSigint() - const config = mockConfig({ storage: ':memory:' }) - const argv = mockArgv(testPath) - const base = mockBase() - const rpc = await createRpc({ argv, sigint }) - - components = { - argv, - sigint, - config, - base, - rpc - } - }) - - after(async () => { - await components.sigint.interupt() - await new Promise(resolve => setTimeout(resolve, 1000)) - await fs.rm(testPath, { recursive: true }) - }) - - it('provides defaults for the config options', async () => { - const m = await network(components) - - assert.deepEqual(m.config.addresses, [ - '/ip4/127.0.0.1/tcp/0', - '/ip4/127.0.0.1/tcp/0/ws' - ]) - - assert.deepEqual(m.config.bootstrap, []) - assert.equal(m.config.private, false) - assert.equal(m.config.serverMode, false) - }) - - it('uses the stores derived form base for helia', async () => { - const m = await network(components) - const cid = CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ') - - assert(!(await m.helia.blockstore.has(cid))) - - await components.base.blockstore.put(cid, new Uint8Array()) - - assert(await m.helia.blockstore.has(cid)) - - assert(!(await m.helia.datastore.has(new Key('/test')))) - - await components.base.datastore.put(new Key('/helia/datastore/test'), new Uint8Array()) - - assert(await m.helia.datastore.has(new Key('/test'))) - }) - - it('is not connectable from outside when private is set to true', async () => { - const config = mockConfig({ private: true }) - - const m = await network({ - ...components, - config - }) - - const libp2p = await createLibp2p({}) - - const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) - - const dialTo = libp2p.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }) - const dialFrom = m.libp2p.dial(libp2p.getMultiaddrs(), { signal: mkSignal() }) - - await Promise.all([ - assert.rejects(async () => dialTo), - assert.rejects(async () => dialFrom) - ]) - - await libp2p.stop() - }) - - it('is connectable from outside when private is set to false', async () => { - const config = mockConfig({ private: false }) - - const m = await network({ - ...components, - config - }) - - const [libp2p1, libp2p2] = await Promise.all([createLibp2p({}), createLibp2p({})]) - - const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) - - await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }), - m.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) - ]) - - await Promise.all([ - libp2p1.stop(), - libp2p2.stop() - ]) - }) - - it('is connectable from inside when private is set to true', async () => { - const config = mockConfig({ private: true }) - - const m = await network({ - ...components, - config - }) - - const create = async (): Promise => createLibp2p({ - psk: components.base.keyManager.getPskKey() - }) - - const [libp2p1, libp2p2] = await Promise.all([create(), create()]) - - const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) - - await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }), - m.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) - ]) - - await Promise.all([ - libp2p1.stop(), - libp2p2.stop() - ]) - }) - - it('bootstraps when a bootstrap peer is set', async () => { - const libp2p = await createLibp2p({}) - - const config = mockConfig({ - private: false, - bootstrap: libp2p.getMultiaddrs().map(a => a.toString()) - }) - - const m = await network({ - ...components, - config - }) - - const peer = await new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, 5000) - - m.libp2p.addEventListener('peer:connect', (a) => { - resolve(a.detail.toBytes()) - }) - }) - - assert.deepEqual(peer, libp2p.peerId.toBytes()) - - await libp2p.stop() - }) - - // Something is failing inside websockets... - it.skip('relays when server mode is set', async () => { - const [libp2p1, libp2p2] = await Promise.all([ - createLibp2p({ addresses: ['/ip4/127.0.0.1/tcp/0'] }), - createLibp2p({ addresses: ['/ip4/127.0.0.1/tcp/0/ws'] }) - ]) - - const m = await network({ - ...components, - config: mockConfig({ - private: false, - serverMode: true - }) - }) - - await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs()), - libp2p2.dial(m.libp2p.getMultiaddrs()) - ]) - - await new Promise(resolve => setTimeout(resolve, 5000)) - - await libp2p1.dial(libp2p2.getMultiaddrs()) - - await Promise.all([ - libp2p1.stop(), - libp2p2.stop() - ]) - }) - - it('libp2p remembers peers with persistant storage', async () => { - const libp2p = await createLibp2p({}) - const sigints = await Promise.all([createSigint(), createSigint()]) - - const create = async (index: number): ReturnType => network({ - ...components, - - sigint: sigints[index], - - base: mockBase({ path: testPath }), - - config: mockConfig({ - private: false - }) - }) - - const m1 = await create(0) - - await libp2p.dial(m1.libp2p.getMultiaddrs()) - - const [peer] = m1.libp2p.getPeers() - - await m1.libp2p.peerStore.save(peer, peer) - - await sigints[0].interupt() - - const m2 = await create(1) - - const peers = await m2.libp2p.peerStore.all() - - assert.deepEqual(peers[0].id.toBytes(), peer.toBytes()) - - await sigints[1].interupt() - await libp2p.stop() - }) - - it('rpc - addresses returns the peers addresses', async () => { - const m = await network(components) - const client = createNetClient(components.argv.socket) - const addresses = await client.rpc.request('addresses', {}) - - assert.deepEqual(addresses, m.libp2p.getMultiaddrs().map(a => a.toString())) - - client.close() - }) - - it('rpc - connections returns the peers connections', async () => { - const libp2p = await createLibp2p({}) - const m = await network(components) - const client = createNetClient(components.argv.socket) - - assert.deepEqual(await client.rpc.request('connections', {}), []) - - await libp2p.dial(m.libp2p.getMultiaddrs()) - - const connections = await client.rpc.request('connections', {}) - - assert.equal(connections.length, 1) - - assert.deepEqual( - connections, - m.libp2p.getConnections().map(a => a.remoteAddr.toString()) - ) - - await libp2p.stop() - client.close() - }) - - it('rpc - connection connects to another peer', async () => { - const libp2p = await createLibp2p({}) - const m = await network(components) - const client = createNetClient(components.argv.socket) - - await client.rpc.request('connect', { address: libp2p.getMultiaddrs()[0] }) - - const connections = m.libp2p.getConnections() - - assert.equal(connections.length, 1) - assert.deepEqual(connections[0].remotePeer.toBytes(), libp2p.peerId.toBytes()) - - await libp2p.stop() - client.close() - }) - - // This should pass but sometimes github workflows can be a bit flakey in terms of peer discovery. - it.skip('rpc - get peers returns a peer hosting content', async () => { - const data = new Uint8Array([0, 1, 2, 3]) - const libp2p = await createLibp2p({}) - const helia = await createHelia({ libp2p }) - const ufs = unixfs(helia) - const m = await network(components) - const client = createNetClient(components.argv.socket) - - await m.libp2p.dial(helia.libp2p.getMultiaddrs()) - - const cid = await ufs.addBytes(data) - await all(helia.pins.add(cid)) - - const result = await client.rpc.request('count-peers', { cids: [cid.toString()] }) - - assert.deepEqual(result, [{ cid: cid.toString(), peers: 1 }]) - - await helia.stop() - await libp2p.stop() - client.close() - }) -}) diff --git a/packages/daemon/test/modules/revisions.spec.ts b/packages/daemon/test/modules/revisions.spec.ts deleted file mode 100644 index 3392b4bb..00000000 --- a/packages/daemon/test/modules/revisions.spec.ts +++ /dev/null @@ -1,460 +0,0 @@ -import assert from 'assert' -import fs from 'fs/promises' -import Path from 'path' -import { unixfs } from '@helia/unixfs' -import { KeyManager } from '@organicdesign/db-key-manager' -import * as testData from '@organicdesign/db-test-utils' -import { importer } from '@organicdesign/db-utils' -import { createNetClient } from '@organicdesign/net-rpc' -import all from 'it-all' -import { CID } from 'multiformats/cid' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createDownloader from '../../src/modules/downloader/index.js' -import createFilesystem from '../../src/modules/filesystem/index.js' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRevisions from '../../src/modules/revisions/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import createTick from '../../src/modules/tick/index.js' -import { createGroup } from '../utils/create-group.js' -import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' -import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' - -describe('revisions', () => { - const testPath = mkTestPath('revisions') - - const create = async (name?: string): Promise<{ - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: Awaited> - filesystem: Awaited> - sigint: Awaited> - tick: Awaited> - revisions: Awaited> - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) - - const tick = await createTick({ config, sigint }) - - const filesystem = await createFilesystem({ - sigint, - base, - rpc, - network, - groups, - downloader, - tick, - config - }) - - const revisions = await createRevisions({ - base, - network, - rpc, - groups, - filesystem, - config, - downloader, - tick - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups, - filesystem, - tick, - revisions - } - } - - before(async () => { - await fs.mkdir(testPath, { recursive: true }) - }) - - after(async () => { - await fs.rm(testPath, { recursive: true }) - }) - - it('returns null when wraping a group that doesn\'t exist', async () => { - const { revisions: m, sigint } = await create() - const r = m.getRevisions(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) - - assert.equal(r, null) - - await sigint.interupt() - }) - - it('wraps a group in revisions', async () => { - const { revisions: m, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const r = m.getRevisions(group) - - assert.notEqual(r, null) - - await sigint.interupt() - }) - - it('creates a revision when a file is added to the filesystem', async () => { - const { revisions: m, filesystem, network, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const r = m.getRevisions(group) - const fs = filesystem.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) - const path = '/test' - - assert(r != null) - assert(fs != null) - - const promise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, 100) - - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) - }) - - await filesystem.uploads.add('put', [group.bytes, path, { - cid: dag[0].bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - const entry = await fs.get(path) - - assert(entry != null) - - await promise - - await new Promise(resolve => setTimeout(resolve, 100)) - - const entries = await all(r.getAll(path)) - - assert.deepEqual(entries, [{ - path, - sequence: 0, - author: groups.welo.identity.id, - - entry: { - cid: entry.cid, - encrypted: entry.encrypted, - timestamp: entry.timestamp, - blocks: entry.blocks, - size: entry.size, - priority: entry.priority - } - }]) - - await sigint.interupt() - }) - - it('rpc - it exports a revision (file)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') - const fs = filesystem.getFileSystem(group) - const path = '/test' - const client = createNetClient(argv.socket) - const sequence = 0 - const dataFile = testData.data[0] - const exportPath = dataFile.generatePath(testPath) - - assert(fs != null) - - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) - - const promise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, 100) - - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) - }) - - await filesystem.uploads.add('put', [group.bytes, path, { - cid: cid.bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - await promise - - await new Promise(resolve => setTimeout(resolve, 100)) - - const response = await client.rpc.request('export-revision', { - group: group.toString(), - path, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc'), - sequence, - outPath: exportPath - }) - - assert.equal(response, null) - - const valid = await dataFile.validate(exportPath) - - assert.equal(valid, true) - - await sigint.interupt() - }) - - it('rpc - it exports a revision (directory)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') - const fs = filesystem.getFileSystem(group) - const rootPath = '/test' - const client = createNetClient(argv.socket) - const sequence = 0 - const outPath = Path.join(testPath, 'export-directory') - - assert(fs != null) - - for (const dataFile of testData.data) { - const virtualPath = dataFile.generatePath(rootPath) - - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) - - const promise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, 100) - - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) - }) - - await filesystem.uploads.add('put', [group.bytes, virtualPath, { - cid: cid.bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - await promise - } - - await new Promise(resolve => setTimeout(resolve, 100)) - - const response = await client.rpc.request('export-revision', { - group: group.toString(), - path: rootPath, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc'), - sequence, - outPath - }) - - assert.equal(response, null) - - for (const dataFile of testData.data) { - const exportPath = dataFile.generatePath(outPath) - const valid = await dataFile.validate(exportPath) - - assert.equal(valid, true) - } - - await sigint.interupt() - }) - - it('rpc - lists a revision (file)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') - const fs = filesystem.getFileSystem(group) - const path = '/test' - const client = createNetClient(argv.socket) - const dataFile = testData.data[0] - - assert(fs != null) - - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) - - const promise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, 100) - - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) - }) - - const before = Date.now() - - await filesystem.uploads.add('put', [group.bytes, path, { - cid: cid.bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - await promise - - await new Promise(resolve => setTimeout(resolve, 100)) - - const response = await client.rpc.request('list', { - group: group.toString(), - path - }) - - assert(Array.isArray(response)) - assert.equal(response.length, 1) - assert.equal(response[0].author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) - assert.equal(response[0].blocks, 1) - assert.equal(response[0].cid, cid.toString()) - assert.equal(response[0].encrypted, false) - assert.equal(response[0].path, path) - assert.equal(response[0].priority, 1) - assert.equal(response[0].revisionStrategy, 'all') - assert.equal(response[0].size, 447) - assert(response[0].timestamp >= before) - assert(response[0].timestamp <= Date.now()) - - await sigint.interupt() - }) - - it('rpc - it lists a revision (directory)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') - const fs = filesystem.getFileSystem(group) - const rootPath = '/test' - const client = createNetClient(argv.socket) - - assert(fs != null) - - const before = Date.now() - - for (const dataFile of testData.data) { - const virtualPath = dataFile.generatePath(rootPath) - - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) - - const promise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, 100) - - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) - }) - - await filesystem.uploads.add('put', [group.bytes, virtualPath, { - cid: cid.bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - await promise - } - - await new Promise(resolve => setTimeout(resolve, 100)) - - const response = await client.rpc.request('list', { - group: group.toString(), - path: rootPath - }) - - assert(Array.isArray(response)) - assert.equal(response.length, 3) - - for (const item of response) { - assert.equal(item.author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) - assert.equal(item.blocks, 1) - assert.equal(item.encrypted, false) - assert.equal(item.priority, 1) - assert.equal(item.revisionStrategy, 'all') - assert(item.timestamp >= before) - assert(item.timestamp <= Date.now()) - } - - for (const dataFile of testData.data) { - const virtualPath = dataFile.generatePath(rootPath) - const item = response.find(d => d.path === virtualPath) - - assert(item != null) - assert(item.cid === dataFile.cid.toString()) - assert(BigInt(item.size) === dataFile.size) - } - - await sigint.interupt() - }) - - it('rpc - read revision', async () => { - const { filesystem, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const ufs = unixfs(network.helia) - const path = '/test' - const data = 'test-data' - - const cid = await ufs.addBytes(uint8ArrayFromString(data)) - - const promise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, 100) - - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) - }) - - await filesystem.uploads.add('put', [group.bytes, path, { - cid: cid.bytes, - encrypted: false, - revisionStrategy: 'all' as const, - priority: 1 - }]) - - await promise - - await new Promise(resolve => setTimeout(resolve, 100)) - - const coreParams = { - group: group.toString(), - path, - sequence: 0, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc') - } - - const read1 = await client.rpc.request('read', coreParams) - - assert.deepEqual(read1, data) - - const read2 = await client.rpc.request('read', { ...coreParams, position: 1 }) - - assert.deepEqual(read2, data.slice(1)) - - const read3 = await client.rpc.request('read', { ...coreParams, length: 3 }) - - assert.deepEqual(read3, data.slice(0, 3)) - - const read4 = await client.rpc.request('read', { ...coreParams, position: 1, length: 3 }) - - assert.deepEqual(read4, data.slice(1, 3 + 1)) - - client.close() - await sigint.interupt() - }) -}) diff --git a/packages/daemon/test/modules/rpc.spec.ts b/packages/daemon/test/modules/rpc.spec.ts deleted file mode 100644 index da8324cc..00000000 --- a/packages/daemon/test/modules/rpc.spec.ts +++ /dev/null @@ -1,61 +0,0 @@ -import assert from 'assert/strict' -import fs from 'fs/promises' -import { createNetClient } from '@organicdesign/net-rpc' -import rpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import type { Provides as Argv } from '../../src/modules/argv/index.js' -import type { Provides as Sigint } from '../../src/modules/sigint/index.js' - -const testPath = mkTestPath('rpc') - -describe('rpc', () => { - let argv: Argv - let sigint: Sigint - - before(async () => { - argv = mockArgv(testPath) - sigint = await createSigint() - - await fs.mkdir(testPath, { recursive: true }) - - await fs.writeFile(argv.key, JSON.stringify({ - key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' - })) - }) - - after(async () => { - await fs.rm(testPath, { recursive: true }) - await sigint.interupt() - }) - - it('adds RPC methods', async () => { - const testData = { key: 'value' } - const returnData = { return: 'return-value' } - - const m = await rpc({ - argv, - sigint - }) - - const client = createNetClient(argv.socket) - - const methodPromise = new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error()) }, 50) - - m.addMethod('test', async params => { - resolve(params) - return returnData - }) - }) - - const returnResult = await client.rpc.request('test', testData) - - assert.deepEqual(returnResult, returnData) - assert.deepEqual(await methodPromise, testData) - - client.close() - }) -}) diff --git a/packages/daemon/test/modules/sigint.spec.ts b/packages/daemon/test/modules/sigint.spec.ts deleted file mode 100644 index 2aa2c68a..00000000 --- a/packages/daemon/test/modules/sigint.spec.ts +++ /dev/null @@ -1,15 +0,0 @@ -import assert from 'assert/strict' -import sigint from '../../src/modules/sigint/index.js' - -describe('sigint', () => { - it('calls the interupt method when interupt gets called', async () => { - const m = await sigint() - - let called = 0 - - m.onInterupt(() => called++) - await m.interupt() - - assert.equal(called, 1) - }) -}) diff --git a/packages/daemon/test/modules/tick.spec.ts b/packages/daemon/test/modules/tick.spec.ts deleted file mode 100644 index bf22fe15..00000000 --- a/packages/daemon/test/modules/tick.spec.ts +++ /dev/null @@ -1,69 +0,0 @@ -import assert from 'assert/strict' -import { type z } from 'zod' -import setupSigint from '../../src/modules/sigint/index.js' -import tick from '../../src/modules/tick/index.js' -import mockConfig from './mock-config.js' - -describe('tick', () => { - it('returns default tick interval', async () => { - const sigint = await setupSigint() - - const m = await tick({ - config: { config: {}, get: (schema: z.AnyZodObject) => schema.parse({}) }, - sigint - }) - - assert.deepEqual(m.config, { tickInterval: 600 }) - - await sigint.interupt() - }) - - it('returns config tick interval', async () => { - const tickInterval = 100 - const sigint = await setupSigint() - - const m = await tick({ - config: mockConfig({ tickInterval }), - sigint - }) - - assert.deepEqual(m.config, { tickInterval }) - - await sigint.interupt() - }) - - it('returns ticks every interval', async () => { - const tickInterval = 5 - const checkTimes = 6 - const sigint = await setupSigint() - - const m = await tick({ - config: mockConfig({ tickInterval }), - sigint - }) - - const before = Date.now() - - await new Promise((resolve, reject) => { - setTimeout(() => { reject(new Error('timeout')) }, (checkTimes + 4) * tickInterval) - let timesCalled = 0 - - m.register(() => { - timesCalled++ - - if (timesCalled >= checkTimes) { - resolve() - } - }) - }) - - const after = Date.now() - - const delta = after - before - - assert(delta < tickInterval * (checkTimes + 2)) - assert(delta > tickInterval * (checkTimes - 2)) - - await sigint.interupt() - }) -}) diff --git a/packages/daemon/test/utils.spec.ts b/packages/daemon/test/utils.spec.ts deleted file mode 100644 index 40ec3425..00000000 --- a/packages/daemon/test/utils.spec.ts +++ /dev/null @@ -1,114 +0,0 @@ -import assert from 'assert/strict' -import { MemoryBlockstore } from 'blockstore-core' -import { type CID } from 'multiformats/cid' -import { MEMORY_MAGIC } from '../src/interface.js' -import { isMemory, encodeAny, decodeAny } from '../src/utils.js' -import { createDag } from './utils/dag.js' -import { walkDag, getDagSize } from '@/modules/filesystem/utils.js' - -describe('isMemory', () => { - it('returns true if the memory magic is passed', () => { - assert(isMemory(MEMORY_MAGIC)) - }) - - it('returns false for any other value than the memory magic', () => { - const values = [ - '/my/path', - 'my-directory', - ':memory', - 'memory', - 'memory:', - ':memory:/my-dir' - ] - - for (const value of values) { - assert(!isMemory(value)) - } - }) -}) - -describe('cbor encoding and decoding', () => { - const data = [ - { - decoded: new Uint8Array([0, 1, 2, 3]), - encoded: new Uint8Array([68, 0, 1, 2, 3]) - }, - - { - decoded: 'str', - encoded: new Uint8Array([99, 115, 116, 114]) - }, - - { - decoded: 9999, - encoded: new Uint8Array([25, 39, 15]) - }, - - { - decoded: [{ test: 'value' }], - encoded: new Uint8Array([129, 161, 100, 116, 101, 115, 116, 101, 118, 97, 108, 117, 101]) - } - ] - - it('encodes any data', () => { - for (const { encoded, decoded } of data) { - assert.deepEqual(encoded, encodeAny(decoded)) - } - }) - - it('decodes any data', () => { - for (const { encoded, decoded } of data) { - assert.deepEqual(decodeAny(encoded), decoded) - } - }) -}) - -describe('walkDag', () => { - let dag: CID[] - let blockstore: MemoryBlockstore - - before(async () => { - blockstore = new MemoryBlockstore() - - dag = await createDag({ blockstore }, 3, 3) - }) - - it('walks over every value of the dag', async () => { - let count = 0 - - for await (const getData of walkDag(blockstore, dag[0])) { - const data = await getData() - - assert(dag.find(cid => cid.equals(data.cid))) - - count++ - } - - assert.equal(count, dag.length) - }) -}) - -describe('getDagSize', () => { - let dag: CID[] - let blockstore: MemoryBlockstore - - before(async () => { - blockstore = new MemoryBlockstore() - - dag = await createDag({ blockstore }, 3, 3) - }) - - it('returns the correct block count for the dag', async () => { - const { blocks } = await getDagSize(blockstore, dag[0]) - - assert.equal(blocks, dag.length) - }) - - it('returns the correct size for the dag', async () => { - const blocks = await Promise.all(dag.map(async cid => blockstore.get(cid))) - const totalSize = blocks.reduce((p, c) => p + c.length, 0) - const { size } = await getDagSize(blockstore, dag[0]) - - assert.equal(size, totalSize) - }) -}) diff --git a/packages/daemon/test/utils/blocks.ts b/packages/daemon/test/utils/blocks.ts deleted file mode 100644 index ae97a7fe..00000000 --- a/packages/daemon/test/utils/blocks.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { CID } from 'multiformats/cid' -import * as raw from 'multiformats/codecs/raw' -import { sha256 } from 'multiformats/hashes/sha2' -import type { Helia } from '@helia/interface' -import type { Blockstore } from 'interface-blockstore' - -export const hashBlock = async (block: Uint8Array, codec?: number): Promise => { - const hash = await sha256.digest(block) - - return CID.createV1(codec ?? raw.code, hash) -} - -export const addBlock = async ({ blockstore }: { blockstore: Blockstore }, block: Uint8Array, codec?: number): Promise => { - const cid = await hashBlock(block, codec) - - await blockstore.put(cid, block) - - return cid -} - -export const createBlocks = async (): Promise> => { - const blocks: Array<{ block: Uint8Array, cid: CID }> = [] - - for (let i = 0; i < 100; i++) { - const block = new Uint8Array([i, i, i]) - const cid = await hashBlock(block) - - blocks.push({ block, cid }) - } - - return blocks -} - -export const addBlocks = async ({ blockstore }: { blockstore: Blockstore }): Promise> => { - const blocks = await createBlocks() - - await Promise.all(blocks.map(async b => addBlock({ blockstore }, b.block))) - - return blocks -} - -export const pinBlocks = async (helia: Helia): Promise => { - const blocks = await addBlocks(helia) - - await Promise.all(blocks.map(({ cid }) => helia.pins.add(cid))) -} diff --git a/packages/daemon/test/utils/create-group.ts b/packages/daemon/test/utils/create-group.ts deleted file mode 100644 index b708735d..00000000 --- a/packages/daemon/test/utils/create-group.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { type CID } from 'multiformats/cid' -import type { Provides as GroupsProvides } from '../../src/modules/groups/index.js' - -export const createGroup = async (m: GroupsProvides, name: string, peers: Uint8Array[] = []): Promise => { - const manifest = await m.welo.determine({ - name, - meta: { type: 'group' }, - access: { - protocol: '/hldb/access/static', - config: { write: [m.welo.identity.id, ...peers] } - } - }) - - await m.groups.add(manifest) - - return manifest.address.cid -} diff --git a/packages/daemon/test/utils/dag.ts b/packages/daemon/test/utils/dag.ts deleted file mode 100644 index a45f1616..00000000 --- a/packages/daemon/test/utils/dag.ts +++ /dev/null @@ -1,44 +0,0 @@ -import * as dagPb from '@ipld/dag-pb' -import { compare as compareUint8Arrays } from 'uint8arrays/compare' -import { fromString as uint8arrayFromString } from 'uint8arrays/from-string' -import { addBlock } from './blocks.js' -import type { Blockstore } from 'interface-blockstore' -import type { CID } from 'multiformats/cid' - -const uniqueNumber = (() => { - let i = 0 - - return () => i++ -})() - -export const createDag = async ({ blockstore }: { blockstore: Blockstore }, depth: number, children: number): Promise => { - if (depth === 0) { - const block = dagPb.encode({ Data: uint8arrayFromString(`level-${depth}-${uniqueNumber()}`), Links: [] }) - const cid = await addBlock({ blockstore }, block, dagPb.code) - - return [cid] - } - - const childrenNodes: CID[][] = [] - - for (let i = 0; i < children; i++) { - childrenNodes.push(await createDag({ blockstore }, depth - 1, children)) - } - - childrenNodes.sort((a, b) => compareUint8Arrays( - uint8arrayFromString(a[0].toString()), - uint8arrayFromString(b[0].toString())) - ) - - const block = dagPb.encode({ - Data: uint8arrayFromString(`level-${depth}-${uniqueNumber()}`), - Links: childrenNodes.map(l => ({ Hash: l[0], Name: l[0].toString() })) - }) - - const cid = await addBlock({ blockstore }, block, dagPb.code) - const allBlocks = childrenNodes.reduce((a, b) => [...a, ...b], []) - - allBlocks.unshift(cid) - - return allBlocks -} diff --git a/packages/daemon/test/utils/generate-key.ts b/packages/daemon/test/utils/generate-key.ts deleted file mode 100644 index 30544ebd..00000000 --- a/packages/daemon/test/utils/generate-key.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { - type KeyData, - parseKeyData, - generateKeyData, - generateMnemonic -} from '@organicdesign/db-key-manager' - -export const generateKey = async (): Promise => { - const mnemonic = generateMnemonic() - const name = generateMnemonic().split(' ')[0] - const rawKeyData = await generateKeyData(mnemonic, name) - - return parseKeyData(rawKeyData) -} diff --git a/packages/daemon/test/utils/paths.ts b/packages/daemon/test/utils/paths.ts deleted file mode 100644 index 9adabb0e..00000000 --- a/packages/daemon/test/utils/paths.ts +++ /dev/null @@ -1,6 +0,0 @@ -import Path from 'path' -import { projectPath } from '@/utils.js' - -export const testPath = Path.join(projectPath, 'packages/daemon/test-out/') - -export const mkTestPath = (name: string): string => Path.join(testPath, name) diff --git a/packages/daemon/tsconfig.json b/packages/daemon/tsconfig.json index a94b36f8..f3691caf 100644 --- a/packages/daemon/tsconfig.json +++ b/packages/daemon/tsconfig.json @@ -13,7 +13,8 @@ } }, "include": [ - "src" + "src", + "test" ], "references": [ { From 31bafa23aa24319915b33973e59f397bf65074d7 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 12:30:47 +1300 Subject: [PATCH 34/62] Linting. --- packages/daemon/src/modules/filesystem/commands/write.ts | 2 +- packages/daemon/src/modules/filesystem/upload-operations.ts | 2 +- packages/daemon/src/modules/revisions/setup.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/daemon/src/modules/filesystem/commands/write.ts b/packages/daemon/src/modules/filesystem/commands/write.ts index f3506f5f..50f4b3c3 100644 --- a/packages/daemon/src/modules/filesystem/commands/write.ts +++ b/packages/daemon/src/modules/filesystem/commands/write.ts @@ -1,4 +1,5 @@ import { unixfs } from '@helia/unixfs' +import { CustomEvent } from '@libp2p/interface' import { Write } from '@organicdesign/db-rpc-interfaces' import all from 'it-all' import { CID } from 'multiformats/cid' @@ -7,7 +8,6 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import type { Context } from '../index.js' import type { Entry } from '../interface.js' import type { ModuleMethod } from '@/interface.js' -import { CustomEvent } from '@libp2p/interface' const command: ModuleMethod = ({ net, helia, events }, context) => { net.rpc.addMethod(Write.name, async (raw: unknown): Promise => { diff --git a/packages/daemon/src/modules/filesystem/upload-operations.ts b/packages/daemon/src/modules/filesystem/upload-operations.ts index fcd63c2c..674472a1 100644 --- a/packages/daemon/src/modules/filesystem/upload-operations.ts +++ b/packages/daemon/src/modules/filesystem/upload-operations.ts @@ -1,5 +1,6 @@ import Path from 'path' import { unixfs } from '@helia/unixfs' +import { CustomEvent } from '@libp2p/interface' import { type RevisionStrategies } from '@organicdesign/db-rpc-interfaces/zod' import all from 'it-all' import { CID } from 'multiformats/cid' @@ -10,7 +11,6 @@ import type { Components } from '@/common/interface.js' import type { Pair } from '@/interface.js' import type { Datastore } from 'interface-datastore' import { OperationManager } from '@/operation-manager.js' -import { CustomEvent } from '@libp2p/interface' export default async (context: Pick, { events, pinManager, helia }: Components, datastore: Datastore): Promise diff --git a/packages/daemon/src/modules/revisions/setup.ts b/packages/daemon/src/modules/revisions/setup.ts index dd232eca..e9b7c201 100644 --- a/packages/daemon/src/modules/revisions/setup.ts +++ b/packages/daemon/src/modules/revisions/setup.ts @@ -1,3 +1,4 @@ +import { CustomEvent } from '@libp2p/interface' import all from 'it-all' import { Revisions } from './revisions.js' import selectRevisions from './select-revisions.js' @@ -5,7 +6,6 @@ import { pathToKey } from './utils.js' import { type Context, logger } from './index.js' import type { Components } from '@/common/interface.js' import type { CID } from 'multiformats/cid' -import { CustomEvent } from '@libp2p/interface' export default async ({ groups, welo, events }: Components): Promise => { const getRevisions = (group: CID): Revisions | null => { From 87c9d7aca8d755d48af52746db4174c3017eac25 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 12:42:01 +1300 Subject: [PATCH 35/62] Linting. --- packages/daemon/src/common/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 52efe390..2e0e1b9e 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -141,7 +141,7 @@ export default async (): Promise => { await welo.stop() await helia.stop() await libp2p.stop() - logger.info('exiting...') + logger.info('exiting...') })().catch(error => { logger.error(error) }) From e271e84650e9671d70ba48036b6cde7114e6e4ff Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 12:55:01 +1300 Subject: [PATCH 36/62] Fit stat commands. --- packages/utils/src/importer.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/utils/src/importer.ts b/packages/utils/src/importer.ts index 7fb03edf..4dd10774 100644 --- a/packages/utils/src/importer.ts +++ b/packages/utils/src/importer.ts @@ -8,9 +8,7 @@ import type { ImportResult } from 'ipfs-unixfs-importer' export const importer = async function * (blockstore: Blockstore, path: string, options: AddOptions = {}): AsyncIterable { const ufs = unixfs({ blockstore }) - - const stat = await fs.lstat(path) - + const stat = await fs.stat(path) const globPattern = stat.isFile() ? path.split('/').pop() ?? '' : '**/*' if (stat.isFile()) { @@ -21,7 +19,7 @@ export const importer = async function * (blockstore: Blockstore, path: string, assert(src.path != null) const rPath = Path.join(path, src.path) - const stat = await fs.lstat(rPath) + const stat = await fs.stat(rPath) if (stat.isDirectory()) { continue From 693fd1d23a1dbaa608d237deee6c00e64fa7c755 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 13:34:50 +1300 Subject: [PATCH 37/62] Fix pin manager's datastore. --- packages/daemon/src/common/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 2e0e1b9e..3d95325e 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -90,7 +90,7 @@ export default async (): Promise => { const heliaPinManager = new HeliaPinManager({ helia, - datastore: extendDatastore(datastore, 'pinManager') + datastore: extendDatastore(datastore, 'heliaPinManager') }) heliaPinManager.events.addEventListener('downloads:added', ({ cid }) => { @@ -117,7 +117,7 @@ export default async (): Promise => { const pinManager = new PinManager({ pinManager: heliaPinManager, - datastore + datastore: extendDatastore(datastore, 'pinManager') }) pinManager.events.addEventListener('reference:removed', ({ key }) => { From 27f8e44ca3e68132dd4767bb5060f6bd946f293d Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 13:46:03 +1300 Subject: [PATCH 38/62] Restore daemon tests. --- packages/daemon/package.json | 3 +- packages/daemon/test/modules/argv.spec.ts | 33 + packages/daemon/test/modules/base.spec.ts | 147 +++++ packages/daemon/test/modules/config.spec.ts | 54 ++ .../daemon/test/modules/downloader.spec.ts | 236 +++++++ .../daemon/test/modules/filesystem.spec.ts | 618 ++++++++++++++++++ packages/daemon/test/modules/groups.spec.ts | 311 +++++++++ packages/daemon/test/modules/mock-argv.ts | 12 + packages/daemon/test/modules/mock-base.ts | 20 + packages/daemon/test/modules/mock-config.ts | 12 + packages/daemon/test/modules/network.spec.ts | 313 +++++++++ .../daemon/test/modules/revisions.spec.ts | 460 +++++++++++++ packages/daemon/test/modules/rpc.spec.ts | 61 ++ packages/daemon/test/modules/sigint.spec.ts | 15 + packages/daemon/test/modules/tick.spec.ts | 69 ++ packages/daemon/test/utils.spec.ts | 114 ++++ packages/daemon/test/utils/blocks.ts | 46 ++ packages/daemon/test/utils/create-group.ts | 17 + packages/daemon/test/utils/dag.ts | 44 ++ packages/daemon/test/utils/generate-key.ts | 14 + packages/daemon/test/utils/paths.ts | 6 + 21 files changed, 2604 insertions(+), 1 deletion(-) create mode 100644 packages/daemon/test/modules/argv.spec.ts create mode 100644 packages/daemon/test/modules/base.spec.ts create mode 100644 packages/daemon/test/modules/config.spec.ts create mode 100644 packages/daemon/test/modules/downloader.spec.ts create mode 100644 packages/daemon/test/modules/filesystem.spec.ts create mode 100644 packages/daemon/test/modules/groups.spec.ts create mode 100644 packages/daemon/test/modules/mock-argv.ts create mode 100644 packages/daemon/test/modules/mock-base.ts create mode 100644 packages/daemon/test/modules/mock-config.ts create mode 100644 packages/daemon/test/modules/network.spec.ts create mode 100644 packages/daemon/test/modules/revisions.spec.ts create mode 100644 packages/daemon/test/modules/rpc.spec.ts create mode 100644 packages/daemon/test/modules/sigint.spec.ts create mode 100644 packages/daemon/test/modules/tick.spec.ts create mode 100644 packages/daemon/test/utils.spec.ts create mode 100644 packages/daemon/test/utils/blocks.ts create mode 100644 packages/daemon/test/utils/create-group.ts create mode 100644 packages/daemon/test/utils/dag.ts create mode 100644 packages/daemon/test/utils/generate-key.ts create mode 100644 packages/daemon/test/utils/paths.ts diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 4551c06b..5bf03e1d 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -19,7 +19,8 @@ "lint": "aegir lint", "dep-check": "aegir dep-check", "prepublishOnly": "npm run build", - "build": "aegir build && tsc-alias" + "build": "aegir build && tsc-alias", + "test": "aegir test -t node -f './dist/test/**/*.spec.js'" }, "author": "Saul Boyd", "license": "GPL-3.0-or-later", diff --git a/packages/daemon/test/modules/argv.spec.ts b/packages/daemon/test/modules/argv.spec.ts new file mode 100644 index 00000000..483f2fa8 --- /dev/null +++ b/packages/daemon/test/modules/argv.spec.ts @@ -0,0 +1,33 @@ +import assert from 'assert/strict' +import Path from 'path' +import argv from '../../src/modules/argv/index.js' +import { projectPath } from '@/utils.js' + +describe('argv', () => { + it('returns defaults for every argv parameter', async () => { + const m = await argv() + + assert.equal(m.key, Path.join(projectPath, 'config/key.json')) + assert.equal(m.config, Path.join(projectPath, 'config/config.json')) + assert.equal(m.socket, '/tmp/server.socket') + }) + + it('returns the value for every argv parameter', async () => { + const key = '/key.json' + const config = '/config.json' + const socket = '/socket' + + process.argv.push('--key') + process.argv.push(key) + process.argv.push('--config') + process.argv.push(config) + process.argv.push('--socket') + process.argv.push(socket) + + const m = await argv() + + assert.equal(m.key, key) + assert.equal(m.config, config) + assert.equal(m.socket, socket) + }) +}) diff --git a/packages/daemon/test/modules/base.spec.ts b/packages/daemon/test/modules/base.spec.ts new file mode 100644 index 00000000..81f9c180 --- /dev/null +++ b/packages/daemon/test/modules/base.spec.ts @@ -0,0 +1,147 @@ +import assert from 'assert/strict' +import fs from 'fs/promises' +import Path from 'path' +import { MemoryBlockstore } from 'blockstore-core' +import { FsBlockstore } from 'blockstore-fs' +import { MemoryDatastore } from 'datastore-core' +import { FsDatastore } from 'datastore-fs' +import { Key } from 'interface-datastore' +import { CID } from 'multiformats/cid' +import { fromString as uint8ArrayFromString } from 'uint8arrays' +import base from '../../src/modules/base/index.js' +import { mkTestPath } from '../utils/paths.js' +import mockArgv from './mock-argv.js' +import mockConfig from './mock-config.js' +import type { Provides as Argv } from '../../src/modules/argv/index.js' +import { extendDatastore } from '@/utils.js' + +const parseStr = (data: string): Uint8Array => uint8ArrayFromString(data, 'base64') +const testPath = mkTestPath('base') + +describe('base', () => { + let argv: Argv + + before(async () => { + argv = mockArgv(testPath) + + await fs.mkdir(Path.join(argv.key, '..'), { recursive: true }) + await fs.mkdir(testPath, { recursive: true }) + + await fs.writeFile(argv.key, JSON.stringify({ + key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', + psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' + })) + }) + + after(async () => { + await fs.rm(testPath, { recursive: true }) + }) + + it('returns the key manager', async () => { + const m = await base({ + config: mockConfig({ storage: ':memory:' }), + argv + }) + + assert.deepEqual( + new Uint8Array(m.keyManager.aesKey), + parseStr('knUGn6uUeQcoxfM1qAtg3F/Njm4bp+GcZK257NZ5AtE') + ) + + assert.deepEqual( + new Uint8Array(m.keyManager.hmacKey), + parseStr('KZkJfNz3bRrn6XHvWYtD4+dXvmhdT4TBhBIkWn8y3jY') + ) + + assert.deepEqual( + (await m.keyManager.getWeloIdentity()).id, + parseStr('CAISIQPSvjmKINqJY5SA/3c+kadFmIsHeTXtTJYlrooZ53DTUg') + ) + + assert.deepEqual( + (await m.keyManager.getPeerId()).toBytes(), + parseStr('ACUIAhIhA5m2/DfXxqi0i+fyYixRaWGirDEVemxUEv8WMZPwFPZB') + ) + + assert.deepEqual( + m.keyManager.getPskKey(), + parseStr('L2tleS9zd2FybS9wc2svMS4wLjAvCi9iYXNlMTYvCjU2ZDNjMTgyODJmMWYxYjFiM2UwNGU0MGRkNWQ4YmY0NGNhZmE4YmM5YzliYzdjNTc3MTZhNzc2NmZhMmM1NTA') + ) + }) + + it('uses memory blockstore when memory is specified', async () => { + const m = await base({ + config: mockConfig({ storage: ':memory:' }), + argv + }) + + assert(m.blockstore instanceof MemoryBlockstore) + }) + + it('uses memory datastore when memory is specified', async () => { + const m = await base({ + config: mockConfig({ storage: ':memory:' }), + argv + }) + + assert(m.datastore instanceof MemoryDatastore) + }) + + it('uses fs blockstore when a path is specified', async () => { + const blockstorePath = Path.join(testPath, 'blockstore') + const testData = uint8ArrayFromString('test') + + const m = await base({ + config: mockConfig({ storage: testPath }), + argv + }) + + assert(m.blockstore instanceof FsBlockstore) + + await m.blockstore.put(CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ'), testData) + + const out = await fs.readdir(blockstorePath, { recursive: true }) + + assert.deepEqual(out, [ + '7I', + '7I/BCIQLASSX2QHMUE4IBHYTTJ3LCICEGM6DOQBZDSN7DTU5RYQ2PEQQX7I.data' + ]) + + const blockData = await fs.readFile(Path.join(blockstorePath, '7I/BCIQLASSX2QHMUE4IBHYTTJ3LCICEGM6DOQBZDSN7DTU5RYQ2PEQQX7I.data')) + + assert.deepEqual(new Uint8Array(blockData), testData) + }) + + it('uses fs datastore when a path is specified', async () => { + const datastorePath = Path.join(testPath, 'datastore') + + const m = await base({ + config: mockConfig({ storage: testPath }), + argv + }) + + assert(m.datastore instanceof FsDatastore) + + await m.datastore.put(new Key('key'), uint8ArrayFromString('value')) + + const subDatastore = extendDatastore(extendDatastore(m.datastore, 'a'), 'b/c') + await subDatastore.put(new Key('d/e'), uint8ArrayFromString('test')) + + const out = await fs.readdir(datastorePath, { recursive: true }) + + assert.deepEqual(out, [ + 'a', + 'key.data', + 'a/b', + 'a/b/c', + 'a/b/c/d', + 'a/b/c/d/e.data' + ]) + + const data1 = await fs.readFile(Path.join(datastorePath, 'key.data')) + const data2 = await fs.readFile(Path.join(datastorePath, 'a/b/c/d/e.data')) + + assert.deepEqual(new Uint8Array(data1), uint8ArrayFromString('value')) + assert.deepEqual(new Uint8Array(data2), uint8ArrayFromString('test')) + }) +}) diff --git a/packages/daemon/test/modules/config.spec.ts b/packages/daemon/test/modules/config.spec.ts new file mode 100644 index 00000000..26c93b57 --- /dev/null +++ b/packages/daemon/test/modules/config.spec.ts @@ -0,0 +1,54 @@ +import assert from 'assert/strict' +import fs from 'fs/promises' +import Path from 'path' +import { z } from 'zod' +import config from '../../src/modules/config/index.js' +import { mkTestPath } from '../utils/paths.js' + +const testPath = mkTestPath('config') +const configPath = Path.join(testPath, 'config.json') + +const configData = { + private: true, + tickInterval: 1000, + storage: ':memory:', + addresses: [ + '/ip4/0.0.0.0/tcp/0' + ], + bootstrap: ['/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'] +} + +before(async () => { + await fs.mkdir(testPath, { recursive: true }) + await fs.writeFile(configPath, JSON.stringify(configData)) +}) + +after(async () => { + await fs.rm(testPath, { recursive: true }) +}) + +describe('config', () => { + it('returns parsed config from the file', async () => { + const m = await config({ + argv: { config: configPath, key: '', socket: '' } + }) + + assert.deepEqual(m.config, configData) + }) + + it('gets config from schema', async () => { + const m = await config({ + argv: { config: configPath, key: '', socket: '' } + }) + + assert.deepEqual( + m.get(z.object({ bootstrap: z.array(z.string()) })), + { bootstrap: configData.bootstrap } + ) + + assert.deepEqual( + m.get(z.object({ tickInterval: z.number(), private: z.boolean() })), + { tickInterval: configData.tickInterval, private: configData.private } + ) + }) +}) diff --git a/packages/daemon/test/modules/downloader.spec.ts b/packages/daemon/test/modules/downloader.spec.ts new file mode 100644 index 00000000..5c459870 --- /dev/null +++ b/packages/daemon/test/modules/downloader.spec.ts @@ -0,0 +1,236 @@ +import assert from 'assert/strict' +import fs from 'fs/promises' +import Path from 'path' +import { KeyManager } from '@organicdesign/db-key-manager' +import { createNetClient } from '@organicdesign/net-rpc' +import { MemoryBlockstore } from 'blockstore-core' +import { CID } from 'multiformats/cid' +import createDownloader from '../../src/modules/downloader/index.js' +import createNetwork from '../../src/modules/network/index.js' +import createRpc from '../../src/modules/rpc/index.js' +import createSigint from '../../src/modules/sigint/index.js' +import { createDag } from '../utils/dag.js' +import { generateKey } from '../utils/generate-key.js' +import { mkTestPath } from '../utils/paths.js' +import mockArgv from './mock-argv.js' +import mockBase from './mock-base.js' +import mockConfig from './mock-config.js' +import type { + Requires as DownloaderComponents, + Provides as DownloaderProvides +} from '../../src/modules/downloader/index.js' + +describe('downloader', () => { + const testPath = mkTestPath('groups') + + const create = async (name?: string): Promise & { + argv: ReturnType + config: ReturnType + rpc: Awaited> + base: ReturnType + network: Awaited> + downloader: DownloaderProvides + }> => { + const path = name == null ? testPath : Path.join(testPath, name) + const keyManager = new KeyManager(await generateKey()) + + await fs.mkdir(path, { recursive: true }) + + const argv = mockArgv(path) + const config = mockConfig({ storage: ':memory:' }) + const sigint = await createSigint() + const rpc = await createRpc({ argv, sigint }) + const base = mockBase({ keyManager }) + const network = await createNetwork({ config, sigint, base, rpc }) + + const downloader = await createDownloader({ + sigint, + base, + rpc, + network, + config + }) + + return { + argv, + config, + sigint, + rpc, + base, + network, + downloader + } + } + + before(async () => { + await fs.mkdir(testPath, { recursive: true }) + }) + + after(async () => { + await fs.rm(testPath, { recursive: true }) + }) + + it('has 20 slots by default', async () => { + const { downloader: m, sigint } = await create() + + assert.equal(m.config.slots, 20) + + await sigint.interupt() + }) + + it('rpc - set priority updates local priority', async () => { + const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' + const path = '/test.txt' + const priority = 50 + const { downloader: m, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const key = Path.join('/', group, path) + + await m.pinManager.put(key, { priority: 1, cid: CID.parse(group) }) + + const response = await client.rpc.request('set-priority', { group, path, priority }) + + assert.equal(response, null) + + const pinData = await m.pinManager.get(key) + + assert(pinData != null) + assert.equal(pinData.priority, priority) + + client.close() + await sigint.interupt() + }) + + it('rpc - get speed', async () => { + const { downloader: m, network, sigint, argv } = await create() + const blockstore = new MemoryBlockstore() + const dag = await createDag({ blockstore }, 2, 2) + const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' + const path = '/test.txt' + const range = 500 + const client = createNetClient(argv.socket) + const key = Path.join('/', group, path) + + await m.pinManager.put(key, { priority: 1, cid: dag[0] }) + + const speed1 = await client.rpc.request('get-speeds', { + cids: [dag[0].toString()], + range + }) + + assert.deepEqual(speed1, [{ cid: dag[0].toString(), speed: 0 }]) + + const value = await blockstore.get(dag[0]) + + await network.helia.blockstore.put(dag[0], value) + await new Promise(resolve => setTimeout(resolve, range / 2)) + + const speed2 = await client.rpc.request('get-speeds', { + cids: [dag[0].toString()], + range + }) + + assert.deepEqual(speed2, [{ cid: dag[0].toString(), speed: value.length / range }]) + + await new Promise(resolve => setTimeout(resolve, range / 2)) + + const values = await Promise.all([ + blockstore.get(dag[1]), + blockstore.get(dag[4]) + ]) + + await Promise.all([ + network.helia.blockstore.put(dag[1], values[0]), + network.helia.blockstore.put(dag[4], values[1]) + ]) + + await new Promise(resolve => setTimeout(resolve, range / 2)) + + const speed3 = await client.rpc.request('get-speeds', { + cids: [dag[0].toString()], + range + }) + + assert.deepEqual(speed3, [{ + cid: dag[0].toString(), + speed: values.reduce((a, c) => c.length + a, 0) / range + }]) + + client.close() + await sigint.interupt() + }) + + it('rpc - get status', async () => { + const { downloader: m, network, sigint, argv } = await create() + const blockstore = new MemoryBlockstore() + const dag = await createDag({ blockstore }, 2, 2) + const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' + const path = '/test.txt' + const client = createNetClient(argv.socket) + const key = Path.join('/', group, path) + + const status1 = await client.rpc.request('get-status', { + cids: [dag[0].toString()] + }) + + assert.deepEqual(status1, [{ + cid: dag[0].toString(), + blocks: 0, + size: 0, + state: 'NOTFOUND' + }]) + + await m.pinManager.put(key, { priority: 1, cid: dag[0] }) + + const status2 = await client.rpc.request('get-status', { + cids: [dag[0].toString()] + }) + + assert.deepEqual(status2, [{ + cid: dag[0].toString(), + blocks: 0, + size: 0, + state: 'DOWNLOADING' + }]) + + const value = await blockstore.get(dag[0]) + + await network.helia.blockstore.put(dag[0], value) + await new Promise(resolve => setTimeout(resolve, 100)) + + const status3 = await client.rpc.request('get-status', { + cids: [dag[0].toString()] + }) + + assert.deepEqual(status3, [{ + cid: dag[0].toString(), + blocks: 1, + size: value.length, + state: 'DOWNLOADING' + }]) + + const values = await Promise.all(dag.map(async cid => { + const value = await blockstore.get(cid) + + await network.helia.blockstore.put(cid, value) + + return value.length + })) + + await new Promise(resolve => setTimeout(resolve, 500)) + + const status4 = await client.rpc.request('get-status', { + cids: [dag[0].toString()] + }) + + assert.deepEqual(status4, [{ + cid: dag[0].toString(), + blocks: dag.length, + size: values.reduce((a, c) => a + c, 0), + state: 'COMPLETED' + }]) + + client.close() + await sigint.interupt() + }) +}) diff --git a/packages/daemon/test/modules/filesystem.spec.ts b/packages/daemon/test/modules/filesystem.spec.ts new file mode 100644 index 00000000..2c1d8896 --- /dev/null +++ b/packages/daemon/test/modules/filesystem.spec.ts @@ -0,0 +1,618 @@ +import assert from 'assert/strict' +import fs from 'fs/promises' +import Path from 'path' +import { unixfs } from '@helia/unixfs' +import { KeyManager } from '@organicdesign/db-key-manager' +import * as testData from '@organicdesign/db-test-utils' +import { importer } from '@organicdesign/db-utils' +import { createNetClient } from '@organicdesign/net-rpc' +import { MemoryBlockstore } from 'blockstore-core' +import all from 'it-all' +import { CID } from 'multiformats/cid' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import createDownloader from '../../src/modules/downloader/index.js' +import createFilesystem from '../../src/modules/filesystem/index.js' +import createGroups from '../../src/modules/groups/index.js' +import createNetwork from '../../src/modules/network/index.js' +import createRpc from '../../src/modules/rpc/index.js' +import createSigint from '../../src/modules/sigint/index.js' +import createTick from '../../src/modules/tick/index.js' +import { createGroup } from '../utils/create-group.js' +import { createDag } from '../utils/dag.js' +import { generateKey } from '../utils/generate-key.js' +import { mkTestPath } from '../utils/paths.js' +import mockArgv from './mock-argv.js' +import mockBase from './mock-base.js' +import mockConfig from './mock-config.js' +import type { Provides as FSProvides } from '../../src/modules/filesystem/index.js' + +describe('filesystem', () => { + const testPath = mkTestPath('filesystem') + + const create = async (name?: string): Promise<{ + argv: ReturnType + config: ReturnType + rpc: Awaited> + base: ReturnType + network: Awaited> + groups: Awaited> + filesystem: FSProvides + sigint: Awaited> + tick: Awaited> + }> => { + const path = name == null ? testPath : Path.join(testPath, name) + + const keyManager = new KeyManager(await generateKey()) + + await fs.mkdir(path, { recursive: true }) + + const argv = mockArgv(path) + const config = mockConfig({ storage: ':memory:' }) + const sigint = await createSigint() + const rpc = await createRpc({ argv, sigint }) + const base = mockBase({ keyManager }) + const network = await createNetwork({ config, sigint, base, rpc }) + + const groups = await createGroups({ + sigint, + base, + rpc, + network + }) + + const downloader = await createDownloader({ + sigint, + base, + rpc, + network, + config + }) + + const tick = await createTick({ config, sigint }) + + const filesystem = await createFilesystem({ + sigint, + base, + rpc, + network, + groups, + downloader, + tick, + config + }) + + return { + argv, + config, + sigint, + rpc, + base, + network, + groups, + filesystem, + tick + } + } + + before(async () => { + await fs.mkdir(testPath, { recursive: true }) + }) + + after(async () => { + await fs.rm(testPath, { recursive: true }) + }) + + it('uses all as the default revision strategy', async () => { + const { filesystem: m, sigint } = await create() + + assert.equal(m.config.defaultRevisionStrategy, 'all') + + await sigint.interupt() + }) + + it('returns null when wraping a group that doesn\'t exist', async () => { + const { filesystem: m, sigint } = await create() + const fs = m.getFileSystem(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) + + assert.equal(fs, null) + + await sigint.interupt() + }) + + it('wraps a group in a filesystem', async () => { + const { filesystem: m, groups, sigint } = await create() + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + + assert.notEqual(fs, null) + + await sigint.interupt() + }) + + it('uploads show in filesystem', async () => { + const { filesystem: m, groups, network, sigint } = await create() + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const dag = await createDag(network.helia, 2, 2) + const path = '/test' + + assert(fs != null) + + const before = Date.now() + + await m.uploads.add('put', [group.bytes, path, { + cid: dag[0].bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + const entry = await fs.get(path) + const values = await Promise.all(dag.map(async d => network.helia.blockstore.get(d))) + const size = values.reduce((a, c) => a + c.length, 0) + + assert(entry != null) + assert.deepEqual(entry.author, groups.welo.identity.id) + assert.equal(entry.blocks, dag.length) + assert.deepEqual(entry.cid, dag[0]) + assert.equal(entry.encrypted, false) + assert.equal(entry.priority, 1) + assert.equal(entry.revisionStrategy, 'all') + assert.equal(entry.sequence, 0) + assert.equal(entry.size, size) + assert(entry.timestamp >= before) + assert(entry.timestamp <= Date.now()) + + await sigint.interupt() + }) + + it('emits the file:added event when the FS was written to', async () => { + const { filesystem: m, groups, network, sigint } = await create() + const group = await createGroup(groups, 'test') + const dag = await createDag(network.helia, 2, 2) + const path = '/test' + + const promise = new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, 100) + + m.events.addEventListener('file:added', () => { resolve() }, { once: true }) + }) + + await m.uploads.add('put', [group.bytes, path, { + cid: dag[0].bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + await promise + + await sigint.interupt() + }) + + it('local settings change FS output', async () => { + const { filesystem: m, groups, network, sigint } = await create() + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const dag = await createDag(network.helia, 2, 2) + const path = '/test' + + assert(fs != null) + + await m.uploads.add('put', [group.bytes, path, { + cid: dag[0].bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + await m.localSettings.set(group, path, { + priority: 100 + }) + + const entry = await fs.get(path) + + assert(entry != null) + assert.equal(entry.priority, 100) + + await sigint.interupt() + }) + + it('rpc - delete (file)', async () => { + const { filesystem: m, groups, network, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const dag = await createDag(network.helia, 2, 2) + const path = '/test' + + assert(fs != null) + + await m.uploads.add('put', [group.bytes, path, { + cid: dag[0].bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + const response = await client.rpc.request('delete', { group: group.toString(), path }) + + assert.deepEqual(response, [{ path, cid: dag[0].toString() }]) + + const entry = await fs.get(path) + + assert.equal(entry, null) + + client.close() + await sigint.interupt() + }) + + it('rpc - delete (directory)', async () => { + const { filesystem: m, groups, network, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const dag = await createDag(network.helia, 2, 2) + const rootPath = '/test' + const paths = [`${rootPath}/file1`, `${rootPath}/file2`, `${rootPath}/sub/file3`] + + assert(fs != null) + + await Promise.all(paths.map(async path => + m.uploads.add('put', [group.bytes, path, { + cid: dag[0].bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + )) + + const response = await client.rpc.request('delete', { group: group.toString(), path: rootPath }) + + assert.deepEqual(response, paths.map(path => ({ path, cid: dag[0].toString() }))) + + const entries = await all(fs.getDir(rootPath)) + + assert.deepEqual(entries, []) + + client.close() + await sigint.interupt() + }) + + it('rpc - edit priority', async () => { + const { filesystem: m, groups, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const path = '/test' + const priority = 50 + + assert(fs != null) + + const response = await client.rpc.request('edit', { group: group.toString(), path, priority }) + + assert.equal(response, null) + + const localSettings = await m.localSettings.get(group, path) + + assert.equal(localSettings.priority, priority) + + client.close() + await sigint.interupt() + }) + + it('rpc - export (file)', async () => { + const { filesystem: m, network, groups, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const rootPath = '/test' + const outPath = Path.join(testPath, 'export-file') + + assert(fs != null) + + await Promise.all(testData.data.map(async data => { + const result = await all(importer(network.helia.blockstore, data.path)) + + await m.uploads.add('put', [group.bytes, data.generatePath(rootPath), { + cid: result[0].cid.bytes, + encrypted: false, + revisionStrategy: 'all', + priority: 1 + }]) + })) + + for (const data of testData.data) { + const exportPath = data.generatePath(outPath) + + const response = await client.rpc.request('export', { + group: group.toString(), + path: data.generatePath(rootPath), + outPath: exportPath + }) + + const valid = await data.validate(exportPath) + + assert.equal(response, null) + assert.equal(valid, true) + } + + client.close() + await sigint.interupt() + }) + + it('rpc - export (directory)', async () => { + const { filesystem: m, network, groups, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const rootPath = '/test' + const outPath = Path.join(testPath, 'export-directory') + + assert(fs != null) + + await Promise.all(testData.data.map(async data => { + const result = await all(importer(network.helia.blockstore, data.path)) + + await m.uploads.add('put', [group.bytes, data.generatePath(rootPath), { + cid: result[0].cid.bytes, + encrypted: false, + revisionStrategy: 'all', + priority: 1 + }]) + })) + + const response = await client.rpc.request('export', { + group: group.toString(), + path: rootPath, + outPath + }) + + assert.equal(response, null) + + for (const data of testData.data) { + const exportPath = data.generatePath(outPath) + const valid = await data.validate(exportPath) + + assert.equal(valid, true) + } + + client.close() + await sigint.interupt() + }) + + it('rpc - import (file)', async () => { + const { filesystem: m, groups, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const rootPath = '/test' + + assert(fs != null) + + await Promise.all(testData.data.map(async data => { + const virtualPath = data.generatePath(rootPath) + + const response = await client.rpc.request('import', { + group: group.toString(), + path: virtualPath, + inPath: data.path + }) + + assert.deepEqual(response, [{ + path: virtualPath, + inPath: data.path, + cid: data.cid.toString() + }]) + + const result = await fs.get(virtualPath) + + assert(result != null) + assert.deepEqual(result.cid, data.cid) + })) + + client.close() + await sigint.interupt() + }) + + it('rpc - import (directory)', async () => { + const { filesystem: m, groups, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const rootPath = '/test' + + assert(fs != null) + + const response = await client.rpc.request('import', { + group: group.toString(), + path: rootPath, + inPath: testData.root + }) + + assert(Array.isArray(response)) + assert.equal(response.length, testData.data.length) + + for (const data of response) { + const dataFile = testData.getDataFile(data.inPath) + + assert(dataFile != null) + assert.equal(data.path, dataFile.generatePath(rootPath)) + assert.equal(data.cid, dataFile.cid.toString()) + } + + const fsResult = await all(fs.getDir(rootPath)) + + assert(Array.isArray(fsResult)) + assert.equal(fsResult.length, testData.data.length) + + for (const dataFile of testData.data) { + const virtualPath = dataFile.generatePath(rootPath) + const r = fsResult.find(r => r.key === virtualPath) + + assert(r != null) + assert.deepEqual(r.value.cid, dataFile.cid) + } + + client.close() + await sigint.interupt() + }) + + it('rpc - list', async () => { + const { filesystem: m, groups, network, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const dag = await createDag(network.helia, 2, 2) + const rootPath = '/test' + const paths = [`${rootPath}/file1`, `${rootPath}/file2`, `${rootPath}/sub/file3`] + + assert(fs != null) + + const before = Date.now() + + await Promise.all(paths.map(async path => + m.uploads.add('put', [group.bytes, path, { + cid: dag[0].bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + )) + + const response = await client.rpc.request('list', { group: group.toString(), path: '/' }) + + assert(Array.isArray(response)) + + const values = await Promise.all(dag.map(async d => network.helia.blockstore.get(d))) + const size = values.reduce((a, c) => a + c.length, 0) + + for (const entry of response) { + assert(entry != null) + assert.equal(entry.author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(entry.blocks, dag.length) + assert.deepEqual(entry.cid, dag[0].toString()) + assert.equal(entry.encrypted, false) + assert.equal(entry.priority, 1) + assert.equal(entry.revisionStrategy, 'all') + assert.equal(entry.size, size) + assert(entry.timestamp >= before) + assert(entry.timestamp <= Date.now()) + } + + client.close() + await sigint.interupt() + }) + + it('rpc - read', async () => { + const { filesystem: m, groups, network, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const ufs = unixfs(network.helia) + const path = '/test' + const data = 'test-data' + + const cid = await ufs.addBytes(uint8ArrayFromString(data)) + + await m.uploads.add('put', [group.bytes, path, { + cid: cid.bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + const read1 = await client.rpc.request('read', { group: group.toString(), path }) + + assert.deepEqual(read1, data) + + const read2 = await client.rpc.request('read', { group: group.toString(), path, position: 1 }) + + assert.deepEqual(read2, data.slice(1)) + + const read3 = await client.rpc.request('read', { group: group.toString(), path, length: 3 }) + + assert.deepEqual(read3, data.slice(0, 3)) + + const read4 = await client.rpc.request('read', { group: group.toString(), path, position: 1, length: 3 }) + + assert.deepEqual(read4, data.slice(1, 3 + 1)) + + client.close() + await sigint.interupt() + }) + + it('rpc - write', async () => { + const { filesystem: m, base, groups, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const fs = m.getFileSystem(group) + const ufs = unixfs({ blockstore: new MemoryBlockstore() }) + const path = '/test' + const data = 'test-data' + const cid = await ufs.addBytes(uint8ArrayFromString(data)) + + assert(fs != null) + + const before = Date.now() + + const write1 = await client.rpc.request('write', { group: group.toString(), path, data }) + + assert.equal(write1, data.length) + + const entry1 = await fs.get(path) + + assert(entry1 != null) + assert.deepEqual(entry1.author, groups.welo.identity.id) + assert.equal(entry1.blocks, 1) + assert.deepEqual(entry1.cid, cid) + assert.equal(entry1.encrypted, false) + assert.equal(entry1.priority, 100) + assert.equal(entry1.revisionStrategy, 'all') + assert.equal(entry1.size, data.length) + assert(entry1.timestamp >= before) + assert(entry1.timestamp <= Date.now()) + + const newData2 = 'your-data-long' + const write2 = await client.rpc.request('write', { group: group.toString(), path, data: newData2 }) + + assert.equal(write2, newData2.length) + + const entry2 = await fs.get(path) + + assert(entry2 != null) + + const value2 = await base.blockstore.get(entry2.cid) + + assert.deepEqual(value2, uint8ArrayFromString(newData2)) + + const newData3 = 'test' + const write3 = await client.rpc.request('write', { group: group.toString(), path, data: newData3, length: newData3.length }) + + assert.equal(write3, newData3.length) + + const entry3 = await fs.get(path) + + assert(entry3 != null) + + const value3 = await base.blockstore.get(entry3.cid) + + assert.deepEqual(value3, uint8ArrayFromString('test-data-long')) + + const newData4 = 'long' + const write4 = await client.rpc.request('write', { group: group.toString(), path, data: newData4, length: newData4.length, position: 5 }) + + assert.equal(write4, newData4.length) + + const entry4 = await fs.get(path) + + assert(entry4 != null) + + const value4 = await base.blockstore.get(entry4.cid) + + assert.deepEqual(value4, uint8ArrayFromString('test-long-long')) + + client.close() + await sigint.interupt() + }) +}) diff --git a/packages/daemon/test/modules/groups.spec.ts b/packages/daemon/test/modules/groups.spec.ts new file mode 100644 index 00000000..f94d84f9 --- /dev/null +++ b/packages/daemon/test/modules/groups.spec.ts @@ -0,0 +1,311 @@ +import assert from 'assert/strict' +import fs from 'fs/promises' +import Path from 'path' +import { KeyManager, parseKeyData } from '@organicdesign/db-key-manager' +import { createNetClient } from '@organicdesign/net-rpc' +import * as cborg from 'cborg' +import all from 'it-all' +import { CID } from 'multiformats/cid' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import createGroups from '../../src/modules/groups/index.js' +import createNetwork from '../../src/modules/network/index.js' +import createRpc from '../../src/modules/rpc/index.js' +import createSigint from '../../src/modules/sigint/index.js' +import { mkTestPath } from '../utils/paths.js' +import mockArgv from './mock-argv.js' +import mockBase from './mock-base.js' +import mockConfig from './mock-config.js' +import type { + Requires as GroupsComponents, + Provides as GroupsProvides +} from '../../src/modules/groups/index.js' + +describe('groups', () => { + const testPath = mkTestPath('groups') + + const mkGroup = async (m: GroupsProvides, name: string, peers: Uint8Array[] = []): Promise => { + const manifest = await m.welo.determine({ + name, + meta: { type: 'group' }, + access: { + protocol: '/hldb/access/static', + config: { write: [m.welo.identity.id, ...peers] } + } + }) + + await m.groups.add(manifest) + + return manifest.address.cid + } + + const create = async (name?: string): Promise & { + argv: ReturnType + config: ReturnType + rpc: Awaited> + base: ReturnType + network: Awaited> + groups: GroupsProvides + }> => { + const path = name == null ? testPath : Path.join(testPath, name) + + const keyManager = name == null + ? undefined + : new KeyManager(parseKeyData({ + key: 'DpGbLiAX4wK4HHtG3DQb8cA6FG2ibv93X4ooZJ5LmMJJ-12FmenN8dbWysuYnzEHzmEF1hod4RGK8NfKFu1SEZ7XM', + psk: '/key/swarm/psk/1.0.0/\n/base16/\n023330a98e30315e2233d4a31a6dc65d741a89f7ce6248e7de40c73995d23157' + })) + + await fs.mkdir(path, { recursive: true }) + + const argv = mockArgv(path) + const config = mockConfig({ storage: ':memory:' }) + const sigint = await createSigint() + const rpc = await createRpc({ argv, sigint }) + const base = mockBase({ keyManager }) + const network = await createNetwork({ config, sigint, base, rpc }) + + const groups = await createGroups({ + sigint, + base, + rpc, + network + }) + + return { + argv, + config, + sigint, + rpc, + base, + network, + groups + } + } + + before(async () => { + await fs.mkdir(testPath, { recursive: true }) + }) + + after(async () => { + await fs.rm(testPath, { recursive: true }) + }) + + it('creates a group', async () => { + const { groups: m, sigint } = await create() + const group = await mkGroup(m, 'test') + + assert(group) + + await sigint.interupt() + }) + + it('tracker puts and checks a group\'s content', async () => { + const { groups: m, sigint } = await create() + const group = await mkGroup(m, 'test') + const database = m.groups.get(group) + + if (database == null) { + throw new Error('group creation failed') + } + + const tracker = m.getTracker(database) + + const key = 'test' + const put = database.store.creators.put(key, 'my-data') + + await database.replica.write(put) + + assert.equal(await tracker.validate(key, put), false) + + await tracker.put(key, put) + + assert.equal(await tracker.validate(key, put), true) + + await sigint.interupt() + }) + + it('tracker processes a group\'s content', async () => { + const { groups: m, sigint } = await create() + const group = await mkGroup(m, 'test') + const database = m.groups.get(group) + + if (database == null) { + throw new Error('group creation failed') + } + + const tracker = m.getTracker(database) + const key = '/test' + const value = 'my-data' + const put = database.store.creators.put(key, value) + + let entries = await all(tracker.process()) + assert.deepEqual(entries, []) + + await database.replica.write(put) + entries = await all(tracker.process()) + assert.deepEqual(entries, [{ key, value: cborg.encode(value) }]) + + entries = await all(tracker.process()) + assert.deepEqual(entries, []) + + await sigint.interupt() + }) + + it('tracker is scope limited', async () => { + const { groups: m, sigint } = await create() + const group = await mkGroup(m, 'test') + const database = m.groups.get(group) + + if (database == null) { + throw new Error('group creation failed') + } + + const tracker = m.getTracker(database) + const key = '/test' + const value = 'my-data' + const put = database.store.creators.put(key, value) + + await database.replica.write(put) + + let entries = await all(tracker.process('/another-key')) + assert.deepEqual(entries, []) + + entries = await all(tracker.process(key)) + assert.deepEqual(entries, [{ key, value: cborg.encode(value) }]) + + await sigint.interupt() + }) + + it('uses the identity from base in welo', async () => { + const { groups: m, sigint, base } = await create() + + assert.deepEqual(m.welo.identity, await base.keyManager.getWeloIdentity()) + + await sigint.interupt() + }) + + it('rpc - id returns the base58btc formatted welo id', async () => { + const { groups: m, sigint, argv } = await create() + const client = createNetClient(argv.socket) + + const id = await client.rpc.request('id', {}) + + assert.equal(uint8ArrayToString(m.welo.identity.id, 'base58btc'), id) + + client.close() + await sigint.interupt() + }) + + it('rpc - create groups creates a group without other peers', async () => { + const { groups: m, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const name = 'test' + + const cid = await client.rpc.request('create-group', { name, peers: [] }) + const group = CID.parse(cid) + const database = m.groups.get(group) + + assert(database != null) + assert.equal(database.manifest.name, name) + assert.deepEqual(database.manifest.access.config?.write, [m.welo.identity.id]) + + client.close() + await sigint.interupt() + }) + + it('rpc - create groups creates a group with other peers', async () => { + const { groups: m, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const name = 'test' + const otherPeer = 'GZsJqUjmbVqZCUMbJoe5ye4xfdKZVPVwBoFFQiyCZYesq6Us5b' + + const cid = await client.rpc.request('create-group', { name, peers: [otherPeer] }) + const group = CID.parse(cid) + const database = m.groups.get(group) + + assert(database != null) + assert.equal(database.manifest.name, name) + assert.deepEqual(database.manifest.access.config?.write, [ + m.welo.identity.id, + uint8ArrayFromString(otherPeer, 'base58btc') + ]) + + client.close() + await sigint.interupt() + }) + + it('rpc - joins an external group', async () => { + const components = await Promise.all([create(), create('server-join-group')]) + const client = createNetClient(components[0].argv.socket) + const name = 'test' + const group = await mkGroup(components[1].groups, name) + + await components[0].network.libp2p.dial(components[1].network.libp2p.getMultiaddrs()) + const res = await client.rpc.request('join-group', { group: group.toString() }) + + assert.equal(res, null) + + const database = components[0].groups.groups.get(group) + + assert(database) + assert.equal(database.manifest.name, name) + + client.close() + await Promise.all(components.map(async c => c.sigint.interupt())) + }) + + it('rpc - list groups', async () => { + const { groups: m, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const name = 'test' + + let groups = await client.rpc.request('list-groups', {}) + + assert.deepEqual(groups, []) + + const group = await mkGroup(m, name) + + groups = await client.rpc.request('list-groups', {}) + + assert.deepEqual(groups, [{ group: group.toString(), name }]) + + client.close() + await sigint.interupt() + }) + + // This fails it github too - seems to think the `server-sync-groups` socket is in use? + it.skip('rpc - sync groups', async () => { + const components = await Promise.all([create(), create('server-sync-groups')]) + const client = createNetClient(components[0].argv.socket) + const key = '/test' + const value = 'test-value' + + const groups = await client.rpc.request('list-groups', {}) + + assert.deepEqual(groups, []) + + const group = await mkGroup(components[1].groups, 'test') + const database = components[1].groups.groups.get(group) + + if (database == null) { + throw new Error('database creation failed') + } + + const put = database.store.creators.put(key, value) + + await database.replica.write(put) + + await components[0].network.libp2p.dial(components[1].network.libp2p.getMultiaddrs()) + await client.rpc.request('join-group', { group: group.toString() }) + await client.rpc.request('sync', {}) + + const index = await database.store.latest() + const result = await database.store.selectors.get(index)(key) + + assert.deepEqual(result, value) + + client.close() + await Promise.all(components.map(async c => c.sigint.interupt())) + }) +}) diff --git a/packages/daemon/test/modules/mock-argv.ts b/packages/daemon/test/modules/mock-argv.ts new file mode 100644 index 00000000..c6da9904 --- /dev/null +++ b/packages/daemon/test/modules/mock-argv.ts @@ -0,0 +1,12 @@ +import Path from 'path' +import type { Provides } from '../../src/modules/argv/index.js' + +const provs = (path: string): Provides => { + return { + key: Path.join(path, 'key.json'), + config: Path.join(path, 'config.json'), + socket: Path.join(path, 'server.socket') + } +} + +export default provs diff --git a/packages/daemon/test/modules/mock-base.ts b/packages/daemon/test/modules/mock-base.ts new file mode 100644 index 00000000..a139b417 --- /dev/null +++ b/packages/daemon/test/modules/mock-base.ts @@ -0,0 +1,20 @@ +import Path from 'path' +import { KeyManager, parseKeyData } from '@organicdesign/db-key-manager' +import { MemoryBlockstore } from 'blockstore-core' +import { FsBlockstore } from 'blockstore-fs' +import { MemoryDatastore } from 'datastore-core' +import { FsDatastore } from 'datastore-fs' +import type { Provides } from '../../src/modules/base/index.js' + +const provs = (config?: { path?: string, keyManager?: KeyManager }): Provides => { + return { + keyManager: config?.keyManager ?? new KeyManager(parseKeyData({ + key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', + psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' + })), + datastore: config?.path == null ? new MemoryDatastore() : new FsDatastore(Path.join(config?.path, 'datastore')), + blockstore: config?.path == null ? new MemoryBlockstore() : new FsBlockstore(Path.join(config?.path, 'blockstore')) + } +} + +export default provs diff --git a/packages/daemon/test/modules/mock-config.ts b/packages/daemon/test/modules/mock-config.ts new file mode 100644 index 00000000..45bf3666 --- /dev/null +++ b/packages/daemon/test/modules/mock-config.ts @@ -0,0 +1,12 @@ +import { type z } from 'zod' +import type { Provides } from '../../src/modules/config/index.js' + +const provs = (config: Record): Provides => ({ + config, + + get (schema: T): z.infer { + return schema.parse(config) + } +}) + +export default provs diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts new file mode 100644 index 00000000..5b83c21d --- /dev/null +++ b/packages/daemon/test/modules/network.spec.ts @@ -0,0 +1,313 @@ +import assert from 'assert/strict' +import fs from 'fs/promises' +import { unixfs } from '@helia/unixfs' +import { createNetClient } from '@organicdesign/net-rpc' +import { createHelia } from 'helia' +import { Key } from 'interface-datastore' +import all from 'it-all' +import { CID } from 'multiformats/cid' +import network from '../../src/modules/network/index.js' +import createRpc from '../../src/modules/rpc/index.js' +import createSigint from '../../src/modules/sigint/index.js' +import { mkTestPath } from '../utils/paths.js' +import mockArgv from './mock-argv.js' +import mockBase from './mock-base.js' +import mockConfig from './mock-config.js' +import type { Requires as NetworkComponents } from '../../src/modules/network/index.js' +import type { Libp2p } from '@libp2p/interface' +import createLibp2p from '@/modules/network/libp2p.js' + +describe('network', () => { + const testPath = mkTestPath('network') + let components: NetworkComponents & { argv: ReturnType } + + before(async () => { + await fs.mkdir(testPath, { recursive: true }) + + const sigint = await createSigint() + const config = mockConfig({ storage: ':memory:' }) + const argv = mockArgv(testPath) + const base = mockBase() + const rpc = await createRpc({ argv, sigint }) + + components = { + argv, + sigint, + config, + base, + rpc + } + }) + + after(async () => { + await components.sigint.interupt() + await new Promise(resolve => setTimeout(resolve, 1000)) + await fs.rm(testPath, { recursive: true }) + }) + + it('provides defaults for the config options', async () => { + const m = await network(components) + + assert.deepEqual(m.config.addresses, [ + '/ip4/127.0.0.1/tcp/0', + '/ip4/127.0.0.1/tcp/0/ws' + ]) + + assert.deepEqual(m.config.bootstrap, []) + assert.equal(m.config.private, false) + assert.equal(m.config.serverMode, false) + }) + + it('uses the stores derived form base for helia', async () => { + const m = await network(components) + const cid = CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ') + + assert(!(await m.helia.blockstore.has(cid))) + + await components.base.blockstore.put(cid, new Uint8Array()) + + assert(await m.helia.blockstore.has(cid)) + + assert(!(await m.helia.datastore.has(new Key('/test')))) + + await components.base.datastore.put(new Key('/helia/datastore/test'), new Uint8Array()) + + assert(await m.helia.datastore.has(new Key('/test'))) + }) + + it('is not connectable from outside when private is set to true', async () => { + const config = mockConfig({ private: true }) + + const m = await network({ + ...components, + config + }) + + const libp2p = await createLibp2p({}) + + const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) + + const dialTo = libp2p.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }) + const dialFrom = m.libp2p.dial(libp2p.getMultiaddrs(), { signal: mkSignal() }) + + await Promise.all([ + assert.rejects(async () => dialTo), + assert.rejects(async () => dialFrom) + ]) + + await libp2p.stop() + }) + + it('is connectable from outside when private is set to false', async () => { + const config = mockConfig({ private: false }) + + const m = await network({ + ...components, + config + }) + + const [libp2p1, libp2p2] = await Promise.all([createLibp2p({}), createLibp2p({})]) + + const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) + + await Promise.all([ + libp2p1.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }), + m.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) + ]) + + await Promise.all([ + libp2p1.stop(), + libp2p2.stop() + ]) + }) + + it('is connectable from inside when private is set to true', async () => { + const config = mockConfig({ private: true }) + + const m = await network({ + ...components, + config + }) + + const create = async (): Promise => createLibp2p({ + psk: components.base.keyManager.getPskKey() + }) + + const [libp2p1, libp2p2] = await Promise.all([create(), create()]) + + const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) + + await Promise.all([ + libp2p1.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }), + m.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) + ]) + + await Promise.all([ + libp2p1.stop(), + libp2p2.stop() + ]) + }) + + it('bootstraps when a bootstrap peer is set', async () => { + const libp2p = await createLibp2p({}) + + const config = mockConfig({ + private: false, + bootstrap: libp2p.getMultiaddrs().map(a => a.toString()) + }) + + const m = await network({ + ...components, + config + }) + + const peer = await new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, 5000) + + m.libp2p.addEventListener('peer:connect', (a) => { + resolve(a.detail.toBytes()) + }) + }) + + assert.deepEqual(peer, libp2p.peerId.toBytes()) + + await libp2p.stop() + }) + + // Something is failing inside websockets... + it.skip('relays when server mode is set', async () => { + const [libp2p1, libp2p2] = await Promise.all([ + createLibp2p({ addresses: ['/ip4/127.0.0.1/tcp/0'] }), + createLibp2p({ addresses: ['/ip4/127.0.0.1/tcp/0/ws'] }) + ]) + + const m = await network({ + ...components, + config: mockConfig({ + private: false, + serverMode: true + }) + }) + + await Promise.all([ + libp2p1.dial(m.libp2p.getMultiaddrs()), + libp2p2.dial(m.libp2p.getMultiaddrs()) + ]) + + await new Promise(resolve => setTimeout(resolve, 5000)) + + await libp2p1.dial(libp2p2.getMultiaddrs()) + + await Promise.all([ + libp2p1.stop(), + libp2p2.stop() + ]) + }) + + it('libp2p remembers peers with persistant storage', async () => { + const libp2p = await createLibp2p({}) + const sigints = await Promise.all([createSigint(), createSigint()]) + + const create = async (index: number): ReturnType => network({ + ...components, + + sigint: sigints[index], + + base: mockBase({ path: testPath }), + + config: mockConfig({ + private: false + }) + }) + + const m1 = await create(0) + + await libp2p.dial(m1.libp2p.getMultiaddrs()) + + const [peer] = m1.libp2p.getPeers() + + await m1.libp2p.peerStore.save(peer, peer) + + await sigints[0].interupt() + + const m2 = await create(1) + + const peers = await m2.libp2p.peerStore.all() + + assert.deepEqual(peers[0].id.toBytes(), peer.toBytes()) + + await sigints[1].interupt() + await libp2p.stop() + }) + + it('rpc - addresses returns the peers addresses', async () => { + const m = await network(components) + const client = createNetClient(components.argv.socket) + const addresses = await client.rpc.request('addresses', {}) + + assert.deepEqual(addresses, m.libp2p.getMultiaddrs().map(a => a.toString())) + + client.close() + }) + + it('rpc - connections returns the peers connections', async () => { + const libp2p = await createLibp2p({}) + const m = await network(components) + const client = createNetClient(components.argv.socket) + + assert.deepEqual(await client.rpc.request('connections', {}), []) + + await libp2p.dial(m.libp2p.getMultiaddrs()) + + const connections = await client.rpc.request('connections', {}) + + assert.equal(connections.length, 1) + + assert.deepEqual( + connections, + m.libp2p.getConnections().map(a => a.remoteAddr.toString()) + ) + + await libp2p.stop() + client.close() + }) + + it('rpc - connection connects to another peer', async () => { + const libp2p = await createLibp2p({}) + const m = await network(components) + const client = createNetClient(components.argv.socket) + + await client.rpc.request('connect', { address: libp2p.getMultiaddrs()[0] }) + + const connections = m.libp2p.getConnections() + + assert.equal(connections.length, 1) + assert.deepEqual(connections[0].remotePeer.toBytes(), libp2p.peerId.toBytes()) + + await libp2p.stop() + client.close() + }) + + // This should pass but sometimes github workflows can be a bit flakey in terms of peer discovery. + it.skip('rpc - get peers returns a peer hosting content', async () => { + const data = new Uint8Array([0, 1, 2, 3]) + const libp2p = await createLibp2p({}) + const helia = await createHelia({ libp2p }) + const ufs = unixfs(helia) + const m = await network(components) + const client = createNetClient(components.argv.socket) + + await m.libp2p.dial(helia.libp2p.getMultiaddrs()) + + const cid = await ufs.addBytes(data) + await all(helia.pins.add(cid)) + + const result = await client.rpc.request('count-peers', { cids: [cid.toString()] }) + + assert.deepEqual(result, [{ cid: cid.toString(), peers: 1 }]) + + await helia.stop() + await libp2p.stop() + client.close() + }) +}) diff --git a/packages/daemon/test/modules/revisions.spec.ts b/packages/daemon/test/modules/revisions.spec.ts new file mode 100644 index 00000000..3392b4bb --- /dev/null +++ b/packages/daemon/test/modules/revisions.spec.ts @@ -0,0 +1,460 @@ +import assert from 'assert' +import fs from 'fs/promises' +import Path from 'path' +import { unixfs } from '@helia/unixfs' +import { KeyManager } from '@organicdesign/db-key-manager' +import * as testData from '@organicdesign/db-test-utils' +import { importer } from '@organicdesign/db-utils' +import { createNetClient } from '@organicdesign/net-rpc' +import all from 'it-all' +import { CID } from 'multiformats/cid' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import createDownloader from '../../src/modules/downloader/index.js' +import createFilesystem from '../../src/modules/filesystem/index.js' +import createGroups from '../../src/modules/groups/index.js' +import createNetwork from '../../src/modules/network/index.js' +import createRevisions from '../../src/modules/revisions/index.js' +import createRpc from '../../src/modules/rpc/index.js' +import createSigint from '../../src/modules/sigint/index.js' +import createTick from '../../src/modules/tick/index.js' +import { createGroup } from '../utils/create-group.js' +import { createDag } from '../utils/dag.js' +import { generateKey } from '../utils/generate-key.js' +import { mkTestPath } from '../utils/paths.js' +import mockArgv from './mock-argv.js' +import mockBase from './mock-base.js' +import mockConfig from './mock-config.js' + +describe('revisions', () => { + const testPath = mkTestPath('revisions') + + const create = async (name?: string): Promise<{ + argv: ReturnType + config: ReturnType + rpc: Awaited> + base: ReturnType + network: Awaited> + groups: Awaited> + filesystem: Awaited> + sigint: Awaited> + tick: Awaited> + revisions: Awaited> + }> => { + const path = name == null ? testPath : Path.join(testPath, name) + + const keyManager = new KeyManager(await generateKey()) + + await fs.mkdir(path, { recursive: true }) + + const argv = mockArgv(path) + const config = mockConfig({ storage: ':memory:' }) + const sigint = await createSigint() + const rpc = await createRpc({ argv, sigint }) + const base = mockBase({ keyManager }) + const network = await createNetwork({ config, sigint, base, rpc }) + + const groups = await createGroups({ + sigint, + base, + rpc, + network + }) + + const downloader = await createDownloader({ + sigint, + base, + rpc, + network, + config + }) + + const tick = await createTick({ config, sigint }) + + const filesystem = await createFilesystem({ + sigint, + base, + rpc, + network, + groups, + downloader, + tick, + config + }) + + const revisions = await createRevisions({ + base, + network, + rpc, + groups, + filesystem, + config, + downloader, + tick + }) + + return { + argv, + config, + sigint, + rpc, + base, + network, + groups, + filesystem, + tick, + revisions + } + } + + before(async () => { + await fs.mkdir(testPath, { recursive: true }) + }) + + after(async () => { + await fs.rm(testPath, { recursive: true }) + }) + + it('returns null when wraping a group that doesn\'t exist', async () => { + const { revisions: m, sigint } = await create() + const r = m.getRevisions(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) + + assert.equal(r, null) + + await sigint.interupt() + }) + + it('wraps a group in revisions', async () => { + const { revisions: m, groups, sigint } = await create() + const group = await createGroup(groups, 'test') + const r = m.getRevisions(group) + + assert.notEqual(r, null) + + await sigint.interupt() + }) + + it('creates a revision when a file is added to the filesystem', async () => { + const { revisions: m, filesystem, network, groups, sigint } = await create() + const group = await createGroup(groups, 'test') + const r = m.getRevisions(group) + const fs = filesystem.getFileSystem(group) + const dag = await createDag(network.helia, 2, 2) + const path = '/test' + + assert(r != null) + assert(fs != null) + + const promise = new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, 100) + + filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + }) + + await filesystem.uploads.add('put', [group.bytes, path, { + cid: dag[0].bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + const entry = await fs.get(path) + + assert(entry != null) + + await promise + + await new Promise(resolve => setTimeout(resolve, 100)) + + const entries = await all(r.getAll(path)) + + assert.deepEqual(entries, [{ + path, + sequence: 0, + author: groups.welo.identity.id, + + entry: { + cid: entry.cid, + encrypted: entry.encrypted, + timestamp: entry.timestamp, + blocks: entry.blocks, + size: entry.size, + priority: entry.priority + } + }]) + + await sigint.interupt() + }) + + it('rpc - it exports a revision (file)', async () => { + const { network, filesystem, groups, sigint, argv } = await create() + const group = await createGroup(groups, 'test') + const fs = filesystem.getFileSystem(group) + const path = '/test' + const client = createNetClient(argv.socket) + const sequence = 0 + const dataFile = testData.data[0] + const exportPath = dataFile.generatePath(testPath) + + assert(fs != null) + + const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + + const promise = new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, 100) + + filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + }) + + await filesystem.uploads.add('put', [group.bytes, path, { + cid: cid.bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + await promise + + await new Promise(resolve => setTimeout(resolve, 100)) + + const response = await client.rpc.request('export-revision', { + group: group.toString(), + path, + author: uint8ArrayToString(groups.welo.identity.id, 'base58btc'), + sequence, + outPath: exportPath + }) + + assert.equal(response, null) + + const valid = await dataFile.validate(exportPath) + + assert.equal(valid, true) + + await sigint.interupt() + }) + + it('rpc - it exports a revision (directory)', async () => { + const { network, filesystem, groups, sigint, argv } = await create() + const group = await createGroup(groups, 'test') + const fs = filesystem.getFileSystem(group) + const rootPath = '/test' + const client = createNetClient(argv.socket) + const sequence = 0 + const outPath = Path.join(testPath, 'export-directory') + + assert(fs != null) + + for (const dataFile of testData.data) { + const virtualPath = dataFile.generatePath(rootPath) + + const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + + const promise = new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, 100) + + filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + }) + + await filesystem.uploads.add('put', [group.bytes, virtualPath, { + cid: cid.bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + await promise + } + + await new Promise(resolve => setTimeout(resolve, 100)) + + const response = await client.rpc.request('export-revision', { + group: group.toString(), + path: rootPath, + author: uint8ArrayToString(groups.welo.identity.id, 'base58btc'), + sequence, + outPath + }) + + assert.equal(response, null) + + for (const dataFile of testData.data) { + const exportPath = dataFile.generatePath(outPath) + const valid = await dataFile.validate(exportPath) + + assert.equal(valid, true) + } + + await sigint.interupt() + }) + + it('rpc - lists a revision (file)', async () => { + const { network, filesystem, groups, sigint, argv } = await create() + const group = await createGroup(groups, 'test') + const fs = filesystem.getFileSystem(group) + const path = '/test' + const client = createNetClient(argv.socket) + const dataFile = testData.data[0] + + assert(fs != null) + + const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + + const promise = new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, 100) + + filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + }) + + const before = Date.now() + + await filesystem.uploads.add('put', [group.bytes, path, { + cid: cid.bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + await promise + + await new Promise(resolve => setTimeout(resolve, 100)) + + const response = await client.rpc.request('list', { + group: group.toString(), + path + }) + + assert(Array.isArray(response)) + assert.equal(response.length, 1) + assert.equal(response[0].author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(response[0].blocks, 1) + assert.equal(response[0].cid, cid.toString()) + assert.equal(response[0].encrypted, false) + assert.equal(response[0].path, path) + assert.equal(response[0].priority, 1) + assert.equal(response[0].revisionStrategy, 'all') + assert.equal(response[0].size, 447) + assert(response[0].timestamp >= before) + assert(response[0].timestamp <= Date.now()) + + await sigint.interupt() + }) + + it('rpc - it lists a revision (directory)', async () => { + const { network, filesystem, groups, sigint, argv } = await create() + const group = await createGroup(groups, 'test') + const fs = filesystem.getFileSystem(group) + const rootPath = '/test' + const client = createNetClient(argv.socket) + + assert(fs != null) + + const before = Date.now() + + for (const dataFile of testData.data) { + const virtualPath = dataFile.generatePath(rootPath) + + const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + + const promise = new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, 100) + + filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + }) + + await filesystem.uploads.add('put', [group.bytes, virtualPath, { + cid: cid.bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + await promise + } + + await new Promise(resolve => setTimeout(resolve, 100)) + + const response = await client.rpc.request('list', { + group: group.toString(), + path: rootPath + }) + + assert(Array.isArray(response)) + assert.equal(response.length, 3) + + for (const item of response) { + assert.equal(item.author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(item.blocks, 1) + assert.equal(item.encrypted, false) + assert.equal(item.priority, 1) + assert.equal(item.revisionStrategy, 'all') + assert(item.timestamp >= before) + assert(item.timestamp <= Date.now()) + } + + for (const dataFile of testData.data) { + const virtualPath = dataFile.generatePath(rootPath) + const item = response.find(d => d.path === virtualPath) + + assert(item != null) + assert(item.cid === dataFile.cid.toString()) + assert(BigInt(item.size) === dataFile.size) + } + + await sigint.interupt() + }) + + it('rpc - read revision', async () => { + const { filesystem, groups, network, sigint, argv } = await create() + const client = createNetClient(argv.socket) + const group = await createGroup(groups, 'test') + const ufs = unixfs(network.helia) + const path = '/test' + const data = 'test-data' + + const cid = await ufs.addBytes(uint8ArrayFromString(data)) + + const promise = new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, 100) + + filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + }) + + await filesystem.uploads.add('put', [group.bytes, path, { + cid: cid.bytes, + encrypted: false, + revisionStrategy: 'all' as const, + priority: 1 + }]) + + await promise + + await new Promise(resolve => setTimeout(resolve, 100)) + + const coreParams = { + group: group.toString(), + path, + sequence: 0, + author: uint8ArrayToString(groups.welo.identity.id, 'base58btc') + } + + const read1 = await client.rpc.request('read', coreParams) + + assert.deepEqual(read1, data) + + const read2 = await client.rpc.request('read', { ...coreParams, position: 1 }) + + assert.deepEqual(read2, data.slice(1)) + + const read3 = await client.rpc.request('read', { ...coreParams, length: 3 }) + + assert.deepEqual(read3, data.slice(0, 3)) + + const read4 = await client.rpc.request('read', { ...coreParams, position: 1, length: 3 }) + + assert.deepEqual(read4, data.slice(1, 3 + 1)) + + client.close() + await sigint.interupt() + }) +}) diff --git a/packages/daemon/test/modules/rpc.spec.ts b/packages/daemon/test/modules/rpc.spec.ts new file mode 100644 index 00000000..da8324cc --- /dev/null +++ b/packages/daemon/test/modules/rpc.spec.ts @@ -0,0 +1,61 @@ +import assert from 'assert/strict' +import fs from 'fs/promises' +import { createNetClient } from '@organicdesign/net-rpc' +import rpc from '../../src/modules/rpc/index.js' +import createSigint from '../../src/modules/sigint/index.js' +import { mkTestPath } from '../utils/paths.js' +import mockArgv from './mock-argv.js' +import type { Provides as Argv } from '../../src/modules/argv/index.js' +import type { Provides as Sigint } from '../../src/modules/sigint/index.js' + +const testPath = mkTestPath('rpc') + +describe('rpc', () => { + let argv: Argv + let sigint: Sigint + + before(async () => { + argv = mockArgv(testPath) + sigint = await createSigint() + + await fs.mkdir(testPath, { recursive: true }) + + await fs.writeFile(argv.key, JSON.stringify({ + key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', + psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' + })) + }) + + after(async () => { + await fs.rm(testPath, { recursive: true }) + await sigint.interupt() + }) + + it('adds RPC methods', async () => { + const testData = { key: 'value' } + const returnData = { return: 'return-value' } + + const m = await rpc({ + argv, + sigint + }) + + const client = createNetClient(argv.socket) + + const methodPromise = new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error()) }, 50) + + m.addMethod('test', async params => { + resolve(params) + return returnData + }) + }) + + const returnResult = await client.rpc.request('test', testData) + + assert.deepEqual(returnResult, returnData) + assert.deepEqual(await methodPromise, testData) + + client.close() + }) +}) diff --git a/packages/daemon/test/modules/sigint.spec.ts b/packages/daemon/test/modules/sigint.spec.ts new file mode 100644 index 00000000..2aa2c68a --- /dev/null +++ b/packages/daemon/test/modules/sigint.spec.ts @@ -0,0 +1,15 @@ +import assert from 'assert/strict' +import sigint from '../../src/modules/sigint/index.js' + +describe('sigint', () => { + it('calls the interupt method when interupt gets called', async () => { + const m = await sigint() + + let called = 0 + + m.onInterupt(() => called++) + await m.interupt() + + assert.equal(called, 1) + }) +}) diff --git a/packages/daemon/test/modules/tick.spec.ts b/packages/daemon/test/modules/tick.spec.ts new file mode 100644 index 00000000..bf22fe15 --- /dev/null +++ b/packages/daemon/test/modules/tick.spec.ts @@ -0,0 +1,69 @@ +import assert from 'assert/strict' +import { type z } from 'zod' +import setupSigint from '../../src/modules/sigint/index.js' +import tick from '../../src/modules/tick/index.js' +import mockConfig from './mock-config.js' + +describe('tick', () => { + it('returns default tick interval', async () => { + const sigint = await setupSigint() + + const m = await tick({ + config: { config: {}, get: (schema: z.AnyZodObject) => schema.parse({}) }, + sigint + }) + + assert.deepEqual(m.config, { tickInterval: 600 }) + + await sigint.interupt() + }) + + it('returns config tick interval', async () => { + const tickInterval = 100 + const sigint = await setupSigint() + + const m = await tick({ + config: mockConfig({ tickInterval }), + sigint + }) + + assert.deepEqual(m.config, { tickInterval }) + + await sigint.interupt() + }) + + it('returns ticks every interval', async () => { + const tickInterval = 5 + const checkTimes = 6 + const sigint = await setupSigint() + + const m = await tick({ + config: mockConfig({ tickInterval }), + sigint + }) + + const before = Date.now() + + await new Promise((resolve, reject) => { + setTimeout(() => { reject(new Error('timeout')) }, (checkTimes + 4) * tickInterval) + let timesCalled = 0 + + m.register(() => { + timesCalled++ + + if (timesCalled >= checkTimes) { + resolve() + } + }) + }) + + const after = Date.now() + + const delta = after - before + + assert(delta < tickInterval * (checkTimes + 2)) + assert(delta > tickInterval * (checkTimes - 2)) + + await sigint.interupt() + }) +}) diff --git a/packages/daemon/test/utils.spec.ts b/packages/daemon/test/utils.spec.ts new file mode 100644 index 00000000..40ec3425 --- /dev/null +++ b/packages/daemon/test/utils.spec.ts @@ -0,0 +1,114 @@ +import assert from 'assert/strict' +import { MemoryBlockstore } from 'blockstore-core' +import { type CID } from 'multiformats/cid' +import { MEMORY_MAGIC } from '../src/interface.js' +import { isMemory, encodeAny, decodeAny } from '../src/utils.js' +import { createDag } from './utils/dag.js' +import { walkDag, getDagSize } from '@/modules/filesystem/utils.js' + +describe('isMemory', () => { + it('returns true if the memory magic is passed', () => { + assert(isMemory(MEMORY_MAGIC)) + }) + + it('returns false for any other value than the memory magic', () => { + const values = [ + '/my/path', + 'my-directory', + ':memory', + 'memory', + 'memory:', + ':memory:/my-dir' + ] + + for (const value of values) { + assert(!isMemory(value)) + } + }) +}) + +describe('cbor encoding and decoding', () => { + const data = [ + { + decoded: new Uint8Array([0, 1, 2, 3]), + encoded: new Uint8Array([68, 0, 1, 2, 3]) + }, + + { + decoded: 'str', + encoded: new Uint8Array([99, 115, 116, 114]) + }, + + { + decoded: 9999, + encoded: new Uint8Array([25, 39, 15]) + }, + + { + decoded: [{ test: 'value' }], + encoded: new Uint8Array([129, 161, 100, 116, 101, 115, 116, 101, 118, 97, 108, 117, 101]) + } + ] + + it('encodes any data', () => { + for (const { encoded, decoded } of data) { + assert.deepEqual(encoded, encodeAny(decoded)) + } + }) + + it('decodes any data', () => { + for (const { encoded, decoded } of data) { + assert.deepEqual(decodeAny(encoded), decoded) + } + }) +}) + +describe('walkDag', () => { + let dag: CID[] + let blockstore: MemoryBlockstore + + before(async () => { + blockstore = new MemoryBlockstore() + + dag = await createDag({ blockstore }, 3, 3) + }) + + it('walks over every value of the dag', async () => { + let count = 0 + + for await (const getData of walkDag(blockstore, dag[0])) { + const data = await getData() + + assert(dag.find(cid => cid.equals(data.cid))) + + count++ + } + + assert.equal(count, dag.length) + }) +}) + +describe('getDagSize', () => { + let dag: CID[] + let blockstore: MemoryBlockstore + + before(async () => { + blockstore = new MemoryBlockstore() + + dag = await createDag({ blockstore }, 3, 3) + }) + + it('returns the correct block count for the dag', async () => { + const { blocks } = await getDagSize(blockstore, dag[0]) + + assert.equal(blocks, dag.length) + }) + + it('returns the correct size for the dag', async () => { + const blocks = await Promise.all(dag.map(async cid => blockstore.get(cid))) + const totalSize = blocks.reduce((p, c) => p + c.length, 0) + const { size } = await getDagSize(blockstore, dag[0]) + + assert.equal(size, totalSize) + }) +}) diff --git a/packages/daemon/test/utils/blocks.ts b/packages/daemon/test/utils/blocks.ts new file mode 100644 index 00000000..ae97a7fe --- /dev/null +++ b/packages/daemon/test/utils/blocks.ts @@ -0,0 +1,46 @@ +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' +import type { Helia } from '@helia/interface' +import type { Blockstore } from 'interface-blockstore' + +export const hashBlock = async (block: Uint8Array, codec?: number): Promise => { + const hash = await sha256.digest(block) + + return CID.createV1(codec ?? raw.code, hash) +} + +export const addBlock = async ({ blockstore }: { blockstore: Blockstore }, block: Uint8Array, codec?: number): Promise => { + const cid = await hashBlock(block, codec) + + await blockstore.put(cid, block) + + return cid +} + +export const createBlocks = async (): Promise> => { + const blocks: Array<{ block: Uint8Array, cid: CID }> = [] + + for (let i = 0; i < 100; i++) { + const block = new Uint8Array([i, i, i]) + const cid = await hashBlock(block) + + blocks.push({ block, cid }) + } + + return blocks +} + +export const addBlocks = async ({ blockstore }: { blockstore: Blockstore }): Promise> => { + const blocks = await createBlocks() + + await Promise.all(blocks.map(async b => addBlock({ blockstore }, b.block))) + + return blocks +} + +export const pinBlocks = async (helia: Helia): Promise => { + const blocks = await addBlocks(helia) + + await Promise.all(blocks.map(({ cid }) => helia.pins.add(cid))) +} diff --git a/packages/daemon/test/utils/create-group.ts b/packages/daemon/test/utils/create-group.ts new file mode 100644 index 00000000..b708735d --- /dev/null +++ b/packages/daemon/test/utils/create-group.ts @@ -0,0 +1,17 @@ +import { type CID } from 'multiformats/cid' +import type { Provides as GroupsProvides } from '../../src/modules/groups/index.js' + +export const createGroup = async (m: GroupsProvides, name: string, peers: Uint8Array[] = []): Promise => { + const manifest = await m.welo.determine({ + name, + meta: { type: 'group' }, + access: { + protocol: '/hldb/access/static', + config: { write: [m.welo.identity.id, ...peers] } + } + }) + + await m.groups.add(manifest) + + return manifest.address.cid +} diff --git a/packages/daemon/test/utils/dag.ts b/packages/daemon/test/utils/dag.ts new file mode 100644 index 00000000..a45f1616 --- /dev/null +++ b/packages/daemon/test/utils/dag.ts @@ -0,0 +1,44 @@ +import * as dagPb from '@ipld/dag-pb' +import { compare as compareUint8Arrays } from 'uint8arrays/compare' +import { fromString as uint8arrayFromString } from 'uint8arrays/from-string' +import { addBlock } from './blocks.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' + +const uniqueNumber = (() => { + let i = 0 + + return () => i++ +})() + +export const createDag = async ({ blockstore }: { blockstore: Blockstore }, depth: number, children: number): Promise => { + if (depth === 0) { + const block = dagPb.encode({ Data: uint8arrayFromString(`level-${depth}-${uniqueNumber()}`), Links: [] }) + const cid = await addBlock({ blockstore }, block, dagPb.code) + + return [cid] + } + + const childrenNodes: CID[][] = [] + + for (let i = 0; i < children; i++) { + childrenNodes.push(await createDag({ blockstore }, depth - 1, children)) + } + + childrenNodes.sort((a, b) => compareUint8Arrays( + uint8arrayFromString(a[0].toString()), + uint8arrayFromString(b[0].toString())) + ) + + const block = dagPb.encode({ + Data: uint8arrayFromString(`level-${depth}-${uniqueNumber()}`), + Links: childrenNodes.map(l => ({ Hash: l[0], Name: l[0].toString() })) + }) + + const cid = await addBlock({ blockstore }, block, dagPb.code) + const allBlocks = childrenNodes.reduce((a, b) => [...a, ...b], []) + + allBlocks.unshift(cid) + + return allBlocks +} diff --git a/packages/daemon/test/utils/generate-key.ts b/packages/daemon/test/utils/generate-key.ts new file mode 100644 index 00000000..30544ebd --- /dev/null +++ b/packages/daemon/test/utils/generate-key.ts @@ -0,0 +1,14 @@ +import { + type KeyData, + parseKeyData, + generateKeyData, + generateMnemonic +} from '@organicdesign/db-key-manager' + +export const generateKey = async (): Promise => { + const mnemonic = generateMnemonic() + const name = generateMnemonic().split(' ')[0] + const rawKeyData = await generateKeyData(mnemonic, name) + + return parseKeyData(rawKeyData) +} diff --git a/packages/daemon/test/utils/paths.ts b/packages/daemon/test/utils/paths.ts new file mode 100644 index 00000000..9adabb0e --- /dev/null +++ b/packages/daemon/test/utils/paths.ts @@ -0,0 +1,6 @@ +import Path from 'path' +import { projectPath } from '@/utils.js' + +export const testPath = Path.join(projectPath, 'packages/daemon/test-out/') + +export const mkTestPath = (name: string): string => Path.join(testPath, name) From b60fc4ddc1f909760120ea2fa38f6d40413a05da Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 13:54:45 +1300 Subject: [PATCH 39/62] Fix group creation util. --- packages/daemon/test/utils/create-group.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/daemon/test/utils/create-group.ts b/packages/daemon/test/utils/create-group.ts index b708735d..063e7487 100644 --- a/packages/daemon/test/utils/create-group.ts +++ b/packages/daemon/test/utils/create-group.ts @@ -1,17 +1,18 @@ import { type CID } from 'multiformats/cid' -import type { Provides as GroupsProvides } from '../../src/modules/groups/index.js' +import type { Groups } from '../../src/common/groups.js' +import type { Welo } from 'welo' -export const createGroup = async (m: GroupsProvides, name: string, peers: Uint8Array[] = []): Promise => { - const manifest = await m.welo.determine({ +export const createGroup = async (groups: Groups, welo: Welo, name: string, peers: Uint8Array[] = []): Promise => { + const manifest = await welo.determine({ name, meta: { type: 'group' }, access: { protocol: '/hldb/access/static', - config: { write: [m.welo.identity.id, ...peers] } + config: { write: [welo.identity.id, ...peers] } } }) - await m.groups.add(manifest) + await groups.add(manifest) return manifest.address.cid } From 744a61a2b6004e3d5006c6df2a099d5b200f7361 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 13:56:49 +1300 Subject: [PATCH 40/62] Fix tick test. --- packages/daemon/test/modules/tick.spec.ts | 45 +++-------------------- 1 file changed, 5 insertions(+), 40 deletions(-) diff --git a/packages/daemon/test/modules/tick.spec.ts b/packages/daemon/test/modules/tick.spec.ts index bf22fe15..07a9e5fb 100644 --- a/packages/daemon/test/modules/tick.spec.ts +++ b/packages/daemon/test/modules/tick.spec.ts @@ -1,46 +1,11 @@ import assert from 'assert/strict' -import { type z } from 'zod' -import setupSigint from '../../src/modules/sigint/index.js' -import tick from '../../src/modules/tick/index.js' -import mockConfig from './mock-config.js' +import { createTick } from '../../src/common/tick.js' describe('tick', () => { - it('returns default tick interval', async () => { - const sigint = await setupSigint() - - const m = await tick({ - config: { config: {}, get: (schema: z.AnyZodObject) => schema.parse({}) }, - sigint - }) - - assert.deepEqual(m.config, { tickInterval: 600 }) - - await sigint.interupt() - }) - - it('returns config tick interval', async () => { - const tickInterval = 100 - const sigint = await setupSigint() - - const m = await tick({ - config: mockConfig({ tickInterval }), - sigint - }) - - assert.deepEqual(m.config, { tickInterval }) - - await sigint.interupt() - }) - it('returns ticks every interval', async () => { const tickInterval = 5 const checkTimes = 6 - const sigint = await setupSigint() - - const m = await tick({ - config: mockConfig({ tickInterval }), - sigint - }) + const tick = await createTick(tickInterval) const before = Date.now() @@ -48,7 +13,7 @@ describe('tick', () => { setTimeout(() => { reject(new Error('timeout')) }, (checkTimes + 4) * tickInterval) let timesCalled = 0 - m.register(() => { + tick.add(() => { timesCalled++ if (timesCalled >= checkTimes) { @@ -59,11 +24,11 @@ describe('tick', () => { const after = Date.now() + await tick.stop() + const delta = after - before assert(delta < tickInterval * (checkTimes + 2)) assert(delta > tickInterval * (checkTimes - 2)) - - await sigint.interupt() }) }) From 8f021c50f8e4d255ab16b6519c2c8c92eb88cfb0 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 13:58:29 +1300 Subject: [PATCH 41/62] Fix argv test. --- packages/daemon/test/modules/argv.spec.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/daemon/test/modules/argv.spec.ts b/packages/daemon/test/modules/argv.spec.ts index 483f2fa8..8db7e163 100644 --- a/packages/daemon/test/modules/argv.spec.ts +++ b/packages/daemon/test/modules/argv.spec.ts @@ -1,15 +1,15 @@ import assert from 'assert/strict' import Path from 'path' -import argv from '../../src/modules/argv/index.js' +import parseArgv from '../../src/common/parse-argv.js' import { projectPath } from '@/utils.js' describe('argv', () => { it('returns defaults for every argv parameter', async () => { - const m = await argv() + const argv = await parseArgv() - assert.equal(m.key, Path.join(projectPath, 'config/key.json')) - assert.equal(m.config, Path.join(projectPath, 'config/config.json')) - assert.equal(m.socket, '/tmp/server.socket') + assert.equal(argv.key, Path.join(projectPath, 'config/key.json')) + assert.equal(argv.config, Path.join(projectPath, 'config/config.json')) + assert.equal(argv.socket, '/tmp/server.socket') }) it('returns the value for every argv parameter', async () => { @@ -24,10 +24,10 @@ describe('argv', () => { process.argv.push('--socket') process.argv.push(socket) - const m = await argv() + const argv = await parseArgv() - assert.equal(m.key, key) - assert.equal(m.config, config) - assert.equal(m.socket, socket) + assert.equal(argv.key, key) + assert.equal(argv.socket, socket) + assert.equal(argv.config, config) }) }) From 9de6337c0d370857833dbfe1d7030556ca1d2cb1 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 14:01:03 +1300 Subject: [PATCH 42/62] Fix config test. --- packages/daemon/test/modules/config.spec.ts | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/packages/daemon/test/modules/config.spec.ts b/packages/daemon/test/modules/config.spec.ts index 26c93b57..bcc32d1a 100644 --- a/packages/daemon/test/modules/config.spec.ts +++ b/packages/daemon/test/modules/config.spec.ts @@ -2,7 +2,7 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' import { z } from 'zod' -import config from '../../src/modules/config/index.js' +import parseConfig from '../../src/common/parse-config.js' import { mkTestPath } from '../utils/paths.js' const testPath = mkTestPath('config') @@ -28,26 +28,16 @@ after(async () => { }) describe('config', () => { - it('returns parsed config from the file', async () => { - const m = await config({ - argv: { config: configPath, key: '', socket: '' } - }) - - assert.deepEqual(m.config, configData) - }) - it('gets config from schema', async () => { - const m = await config({ - argv: { config: configPath, key: '', socket: '' } - }) + const getConfig = await parseConfig(configPath) assert.deepEqual( - m.get(z.object({ bootstrap: z.array(z.string()) })), + getConfig(z.object({ bootstrap: z.array(z.string()) })), { bootstrap: configData.bootstrap } ) assert.deepEqual( - m.get(z.object({ tickInterval: z.number(), private: z.boolean() })), + getConfig(z.object({ tickInterval: z.number(), private: z.boolean() })), { tickInterval: configData.tickInterval, private: configData.private } ) }) From 81c8437a4e6c81d23e03e97ea102112ead282950 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 14:22:56 +1300 Subject: [PATCH 43/62] Make config path optional. --- packages/daemon/src/common/parse-argv.ts | 7 +++---- packages/daemon/src/common/parse-config.ts | 4 ++-- packages/daemon/test/modules/argv.spec.ts | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/daemon/src/common/parse-argv.ts b/packages/daemon/src/common/parse-argv.ts index 83c1f4dc..06490573 100644 --- a/packages/daemon/src/common/parse-argv.ts +++ b/packages/daemon/src/common/parse-argv.ts @@ -6,7 +6,7 @@ import { projectPath } from '@/utils.js' export default async (): Promise<{ socket: string key: string - config: string + config?: string }> => { const argv = await yargs(hideBin(process.argv)) .option({ @@ -26,8 +26,7 @@ export default async (): Promise<{ .option({ config: { alias: 'c', - type: 'string', - default: Path.join(projectPath, 'config/config.json') + type: 'string' } }) .parse() @@ -35,6 +34,6 @@ export default async (): Promise<{ return { socket: Path.resolve(argv.socket), key: Path.resolve(argv.key), - config: Path.resolve(argv.config) + config: argv.config ? Path.resolve(argv.config) : undefined } } diff --git a/packages/daemon/src/common/parse-config.ts b/packages/daemon/src/common/parse-config.ts index 4c37e145..be23bd52 100644 --- a/packages/daemon/src/common/parse-config.ts +++ b/packages/daemon/src/common/parse-config.ts @@ -6,8 +6,8 @@ export interface Provides extends Record { get (shape: T): z.infer } -export default async (path: string): Promise<(shape: T) => z.infer> => { - const raw = await fs.readFile(path, { encoding: 'utf8' }) +export default async (path?: string): Promise<(shape: T) => z.infer> => { + const raw = path != null ? await fs.readFile(path, { encoding: 'utf8' }) : '{}' const config = z.record(z.unknown()).parse(JSON.parse(raw)) const get = (shape: T): z.infer => shape.parse(config) diff --git a/packages/daemon/test/modules/argv.spec.ts b/packages/daemon/test/modules/argv.spec.ts index 8db7e163..24e8ce93 100644 --- a/packages/daemon/test/modules/argv.spec.ts +++ b/packages/daemon/test/modules/argv.spec.ts @@ -8,7 +8,7 @@ describe('argv', () => { const argv = await parseArgv() assert.equal(argv.key, Path.join(projectPath, 'config/key.json')) - assert.equal(argv.config, Path.join(projectPath, 'config/config.json')) + assert.equal(argv.config, undefined) assert.equal(argv.socket, '/tmp/server.socket') }) From bd3cca29f7269ac30badb19d270e0c9028b45efc Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 14:28:17 +1300 Subject: [PATCH 44/62] Replace the abort controller with stop method. --- packages/daemon/src/common/index.ts | 28 +++++++++++-------------- packages/daemon/src/common/interface.ts | 2 +- packages/daemon/src/index.ts | 2 +- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 3d95325e..edb5f4f2 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -30,7 +30,6 @@ export default async (): Promise => { const logger = createLogger('common') const getConfig = await parseConfig(argv.config) const keyManager = await createKeyManager(argv.key) - const controller = new AbortController() const net = await createNetServer(argv.socket) const config = getConfig(Config) const events = new EventTarget() @@ -130,22 +129,19 @@ export default async (): Promise => { logger.error('downloader: ', error) }) - controller.signal.addEventListener('abort', () => { + const stop = async () => { logger.info('cleaning up...') - ;(async () => { - await net.close() - await tick.stop() - await downloader.stop() - await groups.stop() - await welo.stop() - await helia.stop() - await libp2p.stop() - logger.info('exiting...') - })().catch(error => { - logger.error(error) - }) - }) + await net.close() + await tick.stop() + await downloader.stop() + await groups.stop() + await welo.stop() + await helia.stop() + await libp2p.stop() + + logger.info('exiting...') + } const components: Components = { sneakernet, @@ -158,7 +154,7 @@ export default async (): Promise => { tick, downloader, getConfig, - controller, + stop, groups, pinManager, welo, diff --git a/packages/daemon/src/common/interface.ts b/packages/daemon/src/common/interface.ts index e1127b5f..da5cacc3 100644 --- a/packages/daemon/src/common/interface.ts +++ b/packages/daemon/src/common/interface.ts @@ -36,7 +36,7 @@ export interface Components { welo: Welo datastore: Datastore blockstore: Blockstore - controller: AbortController + stop: () => Promise net: NetServer tick: Tick sneakernet: Sneakernet diff --git a/packages/daemon/src/index.ts b/packages/daemon/src/index.ts index f8dd344a..a7828d76 100644 --- a/packages/daemon/src/index.ts +++ b/packages/daemon/src/index.ts @@ -18,7 +18,7 @@ await Promise.all([ ]) process.on('SIGINT', () => { - components.controller.abort() + void components.stop() }) logger.info('started') From cfbaedb889c381ba1ae1aaae789a5c6bb3a7f98d Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 14:43:54 +1300 Subject: [PATCH 45/62] Make key optional. --- packages/daemon/src/common/parse-argv.ts | 10 ++++------ packages/daemon/test/modules/argv.spec.ts | 4 +--- packages/key-manager/src/key-manager.ts | 16 ++++++++++++++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/daemon/src/common/parse-argv.ts b/packages/daemon/src/common/parse-argv.ts index 06490573..d26fb602 100644 --- a/packages/daemon/src/common/parse-argv.ts +++ b/packages/daemon/src/common/parse-argv.ts @@ -1,11 +1,10 @@ import Path from 'path' import { hideBin } from 'yargs/helpers' import yargs from 'yargs/yargs' -import { projectPath } from '@/utils.js' export default async (): Promise<{ socket: string - key: string + key?: string config?: string }> => { const argv = await yargs(hideBin(process.argv)) @@ -19,8 +18,7 @@ export default async (): Promise<{ .option({ key: { alias: 'k', - type: 'string', - default: Path.join(projectPath, 'config/key.json') + type: 'string' } }) .option({ @@ -33,7 +31,7 @@ export default async (): Promise<{ return { socket: Path.resolve(argv.socket), - key: Path.resolve(argv.key), - config: argv.config ? Path.resolve(argv.config) : undefined + key: argv.key != null ? Path.resolve(argv.key) : undefined, + config: argv.config != null ? Path.resolve(argv.config) : undefined } } diff --git a/packages/daemon/test/modules/argv.spec.ts b/packages/daemon/test/modules/argv.spec.ts index 24e8ce93..bedbb0a9 100644 --- a/packages/daemon/test/modules/argv.spec.ts +++ b/packages/daemon/test/modules/argv.spec.ts @@ -1,13 +1,11 @@ import assert from 'assert/strict' -import Path from 'path' import parseArgv from '../../src/common/parse-argv.js' -import { projectPath } from '@/utils.js' describe('argv', () => { it('returns defaults for every argv parameter', async () => { const argv = await parseArgv() - assert.equal(argv.key, Path.join(projectPath, 'config/key.json')) + assert.equal(argv.key, undefined) assert.equal(argv.config, undefined) assert.equal(argv.socket, '/tmp/server.socket') }) diff --git a/packages/key-manager/src/key-manager.ts b/packages/key-manager/src/key-manager.ts index 4bf1b8ae..97d5387b 100644 --- a/packages/key-manager/src/key-manager.ts +++ b/packages/key-manager/src/key-manager.ts @@ -3,7 +3,13 @@ import { keys } from '@libp2p/crypto' import { encode as encodeBlock } from 'multiformats/block' import { sha256 } from 'multiformats/hashes/sha2' import { Identity } from 'welo' -import { importKeyFile, keyToPeerId } from './utils.js' +import { + importKeyFile, + keyToPeerId, + generateKeyData, + generateMnemonic, + parseKeyData +} from './utils.js' import type { KeyData } from './interface.js' import type { PeerId } from '@libp2p/interface' import type { BIP32Interface } from 'bip32' @@ -90,7 +96,13 @@ export class KeyManager { } } -export const createKeyManager = async (path: string): Promise => { +export const createKeyManager = async (path?: string): Promise => { + if (path == null) { + const keys = await generateKeyData(generateMnemonic(), generateMnemonic()[0]) + + return new KeyManager(parseKeyData(keys)) + } + const keys = await importKeyFile(path) return new KeyManager(keys) From e4d4700c57a8178243b8d6ae8292dc839866276d Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 15:38:30 +1300 Subject: [PATCH 46/62] Get network test working again. --- packages/daemon/src/common/index.ts | 33 ++- packages/daemon/src/common/interface.ts | 6 +- packages/daemon/src/common/parse-config.ts | 15 -- packages/daemon/src/index.ts | 11 +- .../daemon/src/modules/filesystem/index.ts | 2 +- .../daemon/src/{common => }/parse-argv.ts | 0 packages/daemon/src/parse-config.ts | 9 + packages/daemon/test/modules/argv.spec.ts | 4 +- packages/daemon/test/modules/network.spec.ts | 193 +++++++----------- packages/daemon/test/modules/tick.spec.ts | 2 +- 10 files changed, 123 insertions(+), 152 deletions(-) delete mode 100644 packages/daemon/src/common/parse-config.ts rename packages/daemon/src/{common => }/parse-argv.ts (100%) create mode 100644 packages/daemon/src/parse-config.ts diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index edb5f4f2..7019ab59 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -10,14 +10,13 @@ import { MemoryDatastore } from 'datastore-core' import { FsDatastore } from 'datastore-fs' import { createHelia } from 'helia' import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' +import { type z } from 'zod' import { createDownloader } from './downloader/index.js' import { EntryTracker } from './entry-tracker.js' import { createGroups } from './groups.js' import handleCommands from './handle-commands.js' import { Config, type Components } from './interface.js' import createLibp2p from './libp2p.js' -import parseArgv from './parse-argv.js' -import parseConfig from './parse-config.js' import { PinManager } from './pin-manager/index.js' import { Sneakernet } from './sneakernet/index.js' import { createTick } from './tick.js' @@ -25,13 +24,24 @@ import type { KeyvalueDB } from '@/interface.js' import { createLogger } from '@/logger.js' import { isMemory, extendDatastore } from '@/utils.js' -export default async (): Promise => { - const argv = await parseArgv() +interface Setup { + socket: string + config: Record + key?: string +} + +export default async (settings: Partial = {}): Promise => { + const setup: Setup = { + socket: settings.socket ?? '/tmp/server.socket', + config: settings.config ?? {}, + key: settings.key + } + const logger = createLogger('common') - const getConfig = await parseConfig(argv.config) - const keyManager = await createKeyManager(argv.key) - const net = await createNetServer(argv.socket) - const config = getConfig(Config) + const parseConfig = (shape: T): z.infer => shape.parse(setup.config) + const keyManager = await createKeyManager(setup.key) + const net = await createNetServer(setup.socket) + const config = parseConfig(Config) const events = new EventTarget() const datastore = isMemory(config.storage) @@ -129,7 +139,7 @@ export default async (): Promise => { logger.error('downloader: ', error) }) - const stop = async () => { + const stop = async (): Promise => { logger.info('cleaning up...') await net.close() @@ -153,13 +163,14 @@ export default async (): Promise => { net, tick, downloader, - getConfig, + parseConfig, stop, groups, pinManager, welo, heliaPinManager, - events + events, + keyManager } handleCommands(components) diff --git a/packages/daemon/src/common/interface.ts b/packages/daemon/src/common/interface.ts index da5cacc3..f14bc9fb 100644 --- a/packages/daemon/src/common/interface.ts +++ b/packages/daemon/src/common/interface.ts @@ -9,6 +9,7 @@ import type { KeyvalueDB } from '@/interface.js' import type { Helia } from '@helia/interface' import type { Libp2p } from '@libp2p/interface' import type HeliaPinManager from '@organicdesign/db-helia-pin-manager' +import type { KeyManager } from '@organicdesign/db-key-manager' import type { NetServer } from '@organicdesign/net-rpc' import type { Blockstore } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' @@ -36,15 +37,16 @@ export interface Components { welo: Welo datastore: Datastore blockstore: Blockstore - stop: () => Promise + stop(): Promise net: NetServer tick: Tick sneakernet: Sneakernet getTracker(keyvalueDB: KeyvalueDB): EntryTracker - getConfig(shape: T): z.infer + parseConfig(shape: T): z.infer downloader: Downloader groups: Groups pinManager: PinManager heliaPinManager: HeliaPinManager events: EventTarget + keyManager: KeyManager } diff --git a/packages/daemon/src/common/parse-config.ts b/packages/daemon/src/common/parse-config.ts deleted file mode 100644 index be23bd52..00000000 --- a/packages/daemon/src/common/parse-config.ts +++ /dev/null @@ -1,15 +0,0 @@ -import fs from 'fs/promises' -import { z } from 'zod' - -export interface Provides extends Record { - config: Record - get (shape: T): z.infer -} - -export default async (path?: string): Promise<(shape: T) => z.infer> => { - const raw = path != null ? await fs.readFile(path, { encoding: 'utf8' }) : '{}' - const config = z.record(z.unknown()).parse(JSON.parse(raw)) - const get = (shape: T): z.infer => shape.parse(config) - - return get -} diff --git a/packages/daemon/src/index.ts b/packages/daemon/src/index.ts index a7828d76..f8ac71f4 100644 --- a/packages/daemon/src/index.ts +++ b/packages/daemon/src/index.ts @@ -1,5 +1,7 @@ import setupCommon from './common/index.js' import { createLogger } from './logger.js' +import parseArgv from './parse-argv.js' +import parseConfig from './parse-config.js' import setupFilesystem from '@/modules/filesystem/index.js' import setupRevisions from '@/modules/revisions/index.js' import setupScheduler from '@/modules/scheduler/index.js' @@ -8,8 +10,15 @@ const logger = createLogger('system') logger.info('starting...') +const argv = await parseArgv() +const config = await parseConfig(argv.config) + // Setup all the modules -const components = await setupCommon() +const components = await setupCommon({ + config, + socket: argv.socket, + key: argv.key +}) await Promise.all([ setupScheduler(components), diff --git a/packages/daemon/src/modules/filesystem/index.ts b/packages/daemon/src/modules/filesystem/index.ts index 47e10705..e95882e3 100644 --- a/packages/daemon/src/modules/filesystem/index.ts +++ b/packages/daemon/src/modules/filesystem/index.ts @@ -33,7 +33,7 @@ export interface Context extends Record { } const module: Module = async (components) => { - const config = components.getConfig(Config) + const config = components.parseConfig(Config) const context = await setup(components, config) for (const setupCommand of [ diff --git a/packages/daemon/src/common/parse-argv.ts b/packages/daemon/src/parse-argv.ts similarity index 100% rename from packages/daemon/src/common/parse-argv.ts rename to packages/daemon/src/parse-argv.ts diff --git a/packages/daemon/src/parse-config.ts b/packages/daemon/src/parse-config.ts new file mode 100644 index 00000000..232edebe --- /dev/null +++ b/packages/daemon/src/parse-config.ts @@ -0,0 +1,9 @@ +import fs from 'fs/promises' +import { z } from 'zod' + +export default async (path?: string): Promise> => { + const raw = path != null ? await fs.readFile(path, { encoding: 'utf8' }) : '{}' + const config = z.record(z.unknown()).parse(JSON.parse(raw)) + + return config +} diff --git a/packages/daemon/test/modules/argv.spec.ts b/packages/daemon/test/modules/argv.spec.ts index bedbb0a9..b31ab2e0 100644 --- a/packages/daemon/test/modules/argv.spec.ts +++ b/packages/daemon/test/modules/argv.spec.ts @@ -1,5 +1,5 @@ import assert from 'assert/strict' -import parseArgv from '../../src/common/parse-argv.js' +import parseArgv from '../../src/parse-argv.js' describe('argv', () => { it('returns defaults for every argv parameter', async () => { @@ -26,6 +26,6 @@ describe('argv', () => { assert.equal(argv.key, key) assert.equal(argv.socket, socket) - assert.equal(argv.config, config) + assert.equal(argv.config, config) }) }) diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts index 5b83c21d..a8c0450c 100644 --- a/packages/daemon/test/modules/network.spec.ts +++ b/packages/daemon/test/modules/network.spec.ts @@ -1,94 +1,71 @@ import assert from 'assert/strict' import fs from 'fs/promises' +import Path from 'path' import { unixfs } from '@helia/unixfs' import { createNetClient } from '@organicdesign/net-rpc' import { createHelia } from 'helia' import { Key } from 'interface-datastore' import all from 'it-all' import { CID } from 'multiformats/cid' -import network from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { Requires as NetworkComponents } from '../../src/modules/network/index.js' import type { Libp2p } from '@libp2p/interface' -import createLibp2p from '@/modules/network/libp2p.js' +import setup from '@/common/index.js' +import { Config, type Components } from '@/common/interface.js' +import createLibp2p from '@/common/libp2p.js' describe('network', () => { const testPath = mkTestPath('network') - let components: NetworkComponents & { argv: ReturnType } + const socket = Path.join(testPath, 'server.socket') - before(async () => { + beforeEach(async () => { await fs.mkdir(testPath, { recursive: true }) - - const sigint = await createSigint() - const config = mockConfig({ storage: ':memory:' }) - const argv = mockArgv(testPath) - const base = mockBase() - const rpc = await createRpc({ argv, sigint }) - - components = { - argv, - sigint, - config, - base, - rpc - } }) - after(async () => { - await components.sigint.interupt() - await new Promise(resolve => setTimeout(resolve, 1000)) + afterEach(async () => { await fs.rm(testPath, { recursive: true }) }) it('provides defaults for the config options', async () => { - const m = await network(components) + const components = await setup({ socket }) + const config = components.parseConfig(Config) - assert.deepEqual(m.config.addresses, [ + assert.deepEqual(config.addresses, [ '/ip4/127.0.0.1/tcp/0', '/ip4/127.0.0.1/tcp/0/ws' ]) - assert.deepEqual(m.config.bootstrap, []) - assert.equal(m.config.private, false) - assert.equal(m.config.serverMode, false) + assert.deepEqual(config.bootstrap, []) + assert.equal(config.private, false) + assert.equal(config.serverMode, false) + + await components.stop() }) it('uses the stores derived form base for helia', async () => { - const m = await network(components) + const components = await setup({ socket }) const cid = CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ') - assert(!(await m.helia.blockstore.has(cid))) + assert(!(await components.helia.blockstore.has(cid))) - await components.base.blockstore.put(cid, new Uint8Array()) + await components.blockstore.put(cid, new Uint8Array()) - assert(await m.helia.blockstore.has(cid)) + assert(await components.helia.blockstore.has(cid)) - assert(!(await m.helia.datastore.has(new Key('/test')))) + assert(!(await components.helia.datastore.has(new Key('/test')))) - await components.base.datastore.put(new Key('/helia/datastore/test'), new Uint8Array()) + await components.datastore.put(new Key('/helia/datastore/test'), new Uint8Array()) - assert(await m.helia.datastore.has(new Key('/test'))) + assert(await components.helia.datastore.has(new Key('/test'))) + + await components.stop() }) it('is not connectable from outside when private is set to true', async () => { - const config = mockConfig({ private: true }) - - const m = await network({ - ...components, - config - }) - + const components = await setup({ socket, config: { private: true } }) const libp2p = await createLibp2p({}) - const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) - - const dialTo = libp2p.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }) - const dialFrom = m.libp2p.dial(libp2p.getMultiaddrs(), { signal: mkSignal() }) + const dialTo = libp2p.dial(components.libp2p.getMultiaddrs(), { signal: mkSignal() }) + const dialFrom = components.libp2p.dial(libp2p.getMultiaddrs(), { signal: mkSignal() }) await Promise.all([ assert.rejects(async () => dialTo), @@ -96,23 +73,18 @@ describe('network', () => { ]) await libp2p.stop() + await components.stop() }) it('is connectable from outside when private is set to false', async () => { - const config = mockConfig({ private: false }) - - const m = await network({ - ...components, - config - }) - + const components = await setup({ socket, config: { private: false } }) const [libp2p1, libp2p2] = await Promise.all([createLibp2p({}), createLibp2p({})]) const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }), - m.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) + libp2p1.dial(components.libp2p.getMultiaddrs(), { signal: mkSignal() }), + components.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) ]) await Promise.all([ @@ -122,15 +94,10 @@ describe('network', () => { }) it('is connectable from inside when private is set to true', async () => { - const config = mockConfig({ private: true }) - - const m = await network({ - ...components, - config - }) + const components = await setup({ socket, config: { private: true } }) const create = async (): Promise => createLibp2p({ - psk: components.base.keyManager.getPskKey() + psk: components.keyManager.getPskKey() }) const [libp2p1, libp2p2] = await Promise.all([create(), create()]) @@ -138,33 +105,31 @@ describe('network', () => { const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }), - m.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) + libp2p1.dial(components.libp2p.getMultiaddrs(), { signal: mkSignal() }), + components.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) ]) await Promise.all([ libp2p1.stop(), - libp2p2.stop() + libp2p2.stop(), + components.stop() ]) }) it('bootstraps when a bootstrap peer is set', async () => { const libp2p = await createLibp2p({}) - - const config = mockConfig({ - private: false, - bootstrap: libp2p.getMultiaddrs().map(a => a.toString()) - }) - - const m = await network({ - ...components, - config + const components = await setup({ + socket, + config: { + private: false, + bootstrap: libp2p.getMultiaddrs().map(a => a.toString()) + } }) const peer = await new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 5000) - m.libp2p.addEventListener('peer:connect', (a) => { + components.libp2p.addEventListener('peer:connect', (a) => { resolve(a.detail.toBytes()) }) }) @@ -181,17 +146,17 @@ describe('network', () => { createLibp2p({ addresses: ['/ip4/127.0.0.1/tcp/0/ws'] }) ]) - const m = await network({ - ...components, - config: mockConfig({ + const components = await setup({ + socket, + config: { private: false, serverMode: true - }) + } }) await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs()), - libp2p2.dial(m.libp2p.getMultiaddrs()) + libp2p1.dial(components.libp2p.getMultiaddrs()), + libp2p2.dial(components.libp2p.getMultiaddrs()) ]) await new Promise(resolve => setTimeout(resolve, 5000)) @@ -206,58 +171,47 @@ describe('network', () => { it('libp2p remembers peers with persistant storage', async () => { const libp2p = await createLibp2p({}) - const sigints = await Promise.all([createSigint(), createSigint()]) - - const create = async (index: number): ReturnType => network({ - ...components, - - sigint: sigints[index], - - base: mockBase({ path: testPath }), - - config: mockConfig({ - private: false - }) - }) + const start = async (): Promise => setup({ socket, config: { private: false, storage: testPath } }) - const m1 = await create(0) + let components = await start() - await libp2p.dial(m1.libp2p.getMultiaddrs()) + await libp2p.dial(components.libp2p.getMultiaddrs()) - const [peer] = m1.libp2p.getPeers() + const [peer] = components.libp2p.getPeers() - await m1.libp2p.peerStore.save(peer, peer) + await components.libp2p.peerStore.save(peer, peer) - await sigints[0].interupt() + await components.stop() - const m2 = await create(1) + components = await start() - const peers = await m2.libp2p.peerStore.all() + const peers = await components.libp2p.peerStore.all() assert.deepEqual(peers[0].id.toBytes(), peer.toBytes()) - await sigints[1].interupt() + await components.stop() await libp2p.stop() }) it('rpc - addresses returns the peers addresses', async () => { - const m = await network(components) - const client = createNetClient(components.argv.socket) + const components = await setup({ socket, config: { private: false } }) + const client = createNetClient(socket) const addresses = await client.rpc.request('addresses', {}) - assert.deepEqual(addresses, m.libp2p.getMultiaddrs().map(a => a.toString())) + assert.deepEqual(addresses, components.libp2p.getMultiaddrs().map(a => a.toString())) client.close() + await components.stop() }) it('rpc - connections returns the peers connections', async () => { const libp2p = await createLibp2p({}) - const m = await network(components) - const client = createNetClient(components.argv.socket) + const components = await setup({ socket, config: { private: false } }) + const client = createNetClient(socket) assert.deepEqual(await client.rpc.request('connections', {}), []) - await libp2p.dial(m.libp2p.getMultiaddrs()) + await libp2p.dial(components.libp2p.getMultiaddrs()) const connections = await client.rpc.request('connections', {}) @@ -265,21 +219,22 @@ describe('network', () => { assert.deepEqual( connections, - m.libp2p.getConnections().map(a => a.remoteAddr.toString()) + components.libp2p.getConnections().map(a => a.remoteAddr.toString()) ) await libp2p.stop() client.close() + await components.stop() }) it('rpc - connection connects to another peer', async () => { const libp2p = await createLibp2p({}) - const m = await network(components) - const client = createNetClient(components.argv.socket) + const components = await setup({ socket, config: { private: false } }) + const client = createNetClient(socket) await client.rpc.request('connect', { address: libp2p.getMultiaddrs()[0] }) - const connections = m.libp2p.getConnections() + const connections = components.libp2p.getConnections() assert.equal(connections.length, 1) assert.deepEqual(connections[0].remotePeer.toBytes(), libp2p.peerId.toBytes()) @@ -294,10 +249,10 @@ describe('network', () => { const libp2p = await createLibp2p({}) const helia = await createHelia({ libp2p }) const ufs = unixfs(helia) - const m = await network(components) - const client = createNetClient(components.argv.socket) + const components = await setup({ socket, config: { private: false } }) + const client = createNetClient(socket) - await m.libp2p.dial(helia.libp2p.getMultiaddrs()) + await components.libp2p.dial(helia.libp2p.getMultiaddrs()) const cid = await ufs.addBytes(data) await all(helia.pins.add(cid)) diff --git a/packages/daemon/test/modules/tick.spec.ts b/packages/daemon/test/modules/tick.spec.ts index 07a9e5fb..02b1fa04 100644 --- a/packages/daemon/test/modules/tick.spec.ts +++ b/packages/daemon/test/modules/tick.spec.ts @@ -24,7 +24,7 @@ describe('tick', () => { const after = Date.now() - await tick.stop() + await tick.stop() const delta = after - before From c2056eb33854e4bd9035ad994f72218832ee69b2 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 15:53:59 +1300 Subject: [PATCH 47/62] Fix config test. --- packages/daemon/test/modules/config.spec.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/daemon/test/modules/config.spec.ts b/packages/daemon/test/modules/config.spec.ts index bcc32d1a..c47bc0b8 100644 --- a/packages/daemon/test/modules/config.spec.ts +++ b/packages/daemon/test/modules/config.spec.ts @@ -2,8 +2,9 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' import { z } from 'zod' -import parseConfig from '../../src/common/parse-config.js' +import parseConfig from '../../src/parse-config.js' import { mkTestPath } from '../utils/paths.js' +import setup from '@/common/index.js' const testPath = mkTestPath('config') const configPath = Path.join(testPath, 'config.json') @@ -28,17 +29,25 @@ after(async () => { }) describe('config', () => { - it('gets config from schema', async () => { - const getConfig = await parseConfig(configPath) + it('gets config from file', async () => { + const config = await parseConfig(configPath) + + assert.deepEqual(config, configData) + }) + + it('parses config from schema', async () => { + const components = await setup({ config: configData, socket: Path.join(testPath, 'server.socket') }) assert.deepEqual( - getConfig(z.object({ bootstrap: z.array(z.string()) })), + components.parseConfig(z.object({ bootstrap: z.array(z.string()) })), { bootstrap: configData.bootstrap } ) assert.deepEqual( - getConfig(z.object({ tickInterval: z.number(), private: z.boolean() })), + components.parseConfig(z.object({ tickInterval: z.number(), private: z.boolean() })), { tickInterval: configData.tickInterval, private: configData.private } ) - }) + + await components.stop() + }) }) From eed2234df4e3b23e3d1a4ec2637d30bd6d72581a Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 15:54:20 +1300 Subject: [PATCH 48/62] Stop all the components in network test. --- packages/daemon/test/modules/network.spec.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts index a8c0450c..78a9418b 100644 --- a/packages/daemon/test/modules/network.spec.ts +++ b/packages/daemon/test/modules/network.spec.ts @@ -89,7 +89,8 @@ describe('network', () => { await Promise.all([ libp2p1.stop(), - libp2p2.stop() + libp2p2.stop(), + components.stop() ]) }) @@ -137,6 +138,7 @@ describe('network', () => { assert.deepEqual(peer, libp2p.peerId.toBytes()) await libp2p.stop() + await components.stop() }) // Something is failing inside websockets... @@ -165,7 +167,8 @@ describe('network', () => { await Promise.all([ libp2p1.stop(), - libp2p2.stop() + libp2p2.stop(), + components.stop() ]) }) @@ -240,6 +243,7 @@ describe('network', () => { assert.deepEqual(connections[0].remotePeer.toBytes(), libp2p.peerId.toBytes()) await libp2p.stop() + await components.stop() client.close() }) @@ -263,6 +267,7 @@ describe('network', () => { await helia.stop() await libp2p.stop() + await components.stop() client.close() }) }) From d3bfbbe13ded2d15844aff38135fdd13efd8079d Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 16:13:56 +1300 Subject: [PATCH 49/62] Fix revisions test. --- packages/daemon/test/modules/network.spec.ts | 4 +- .../daemon/test/modules/revisions.spec.ts | 205 ++++++------------ packages/daemon/test/utils/create-group.ts | 5 +- 3 files changed, 71 insertions(+), 143 deletions(-) diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts index 78a9418b..d5bd8ab3 100644 --- a/packages/daemon/test/modules/network.spec.ts +++ b/packages/daemon/test/modules/network.spec.ts @@ -188,9 +188,9 @@ describe('network', () => { components = await start() - const peers = await components.libp2p.peerStore.all() + const [saved] = await components.libp2p.peerStore.all() - assert.deepEqual(peers[0].id.toBytes(), peer.toBytes()) + assert.deepEqual(saved.id.toBytes(), peer.toBytes()) await components.stop() await libp2p.stop() diff --git a/packages/daemon/test/modules/revisions.spec.ts b/packages/daemon/test/modules/revisions.spec.ts index 3392b4bb..9f95ccdb 100644 --- a/packages/daemon/test/modules/revisions.spec.ts +++ b/packages/daemon/test/modules/revisions.spec.ts @@ -2,7 +2,6 @@ import assert from 'assert' import fs from 'fs/promises' import Path from 'path' import { unixfs } from '@helia/unixfs' -import { KeyManager } from '@organicdesign/db-key-manager' import * as testData from '@organicdesign/db-test-utils' import { importer } from '@organicdesign/db-utils' import { createNetClient } from '@organicdesign/net-rpc' @@ -10,101 +9,31 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createDownloader from '../../src/modules/downloader/index.js' -import createFilesystem from '../../src/modules/filesystem/index.js' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRevisions from '../../src/modules/revisions/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import createTick from '../../src/modules/tick/index.js' +import setup from '@/common/index.js' +import setupRevisions from '@/modules/revisions/index.js' +import setupFilesystem from '@/modules/filesystem/index.js' +import type { Context as RevisionsContext } from '@/modules/revisions/index.js' +import type { Context as FilesystemContext } from '@/modules/filesystem/index.js' +import { mkTestPath } from '../utils/paths.js' +import type { Components } from '@/common/interface.js' import { createGroup } from '../utils/create-group.js' import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' -import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' describe('revisions', () => { const testPath = mkTestPath('revisions') const create = async (name?: string): Promise<{ - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: Awaited> - filesystem: Awaited> - sigint: Awaited> - tick: Awaited> - revisions: Awaited> - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) - - const tick = await createTick({ config, sigint }) - - const filesystem = await createFilesystem({ - sigint, - base, - rpc, - network, - groups, - downloader, - tick, - config - }) - - const revisions = await createRevisions({ - base, - network, - rpc, - groups, - filesystem, - config, - downloader, - tick - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups, - filesystem, - tick, - revisions - } + revisions: RevisionsContext + filesystem: FilesystemContext + components: Components + socket: string + }> => { + const socket = Path.join(testPath, `${name ?? 'server'}.socket`) + const components = await setup({ socket }) + const filesystem = await setupFilesystem(components) + const revisions = await setupRevisions(components) + + return { filesystem, revisions, components, socket } } before(async () => { @@ -116,30 +45,30 @@ describe('revisions', () => { }) it('returns null when wraping a group that doesn\'t exist', async () => { - const { revisions: m, sigint } = await create() - const r = m.getRevisions(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) + const { revisions, components } = await create() + const r = revisions.getRevisions(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) assert.equal(r, null) - await sigint.interupt() + await components.stop() }) it('wraps a group in revisions', async () => { - const { revisions: m, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const r = m.getRevisions(group) + const { revisions, components } = await create() + const group = await createGroup(components, 'test') + const r = revisions.getRevisions(group) assert.notEqual(r, null) - await sigint.interupt() + await components.stop() }) it('creates a revision when a file is added to the filesystem', async () => { - const { revisions: m, filesystem, network, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const r = m.getRevisions(group) + const { revisions, filesystem, components } = await create() + const group = await createGroup(components, 'test') + const r = revisions.getRevisions(group) const fs = filesystem.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const dag = await createDag(components.helia, 2, 2) const path = '/test' assert(r != null) @@ -148,7 +77,7 @@ describe('revisions', () => { const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, path, { @@ -171,7 +100,7 @@ describe('revisions', () => { assert.deepEqual(entries, [{ path, sequence: 0, - author: groups.welo.identity.id, + author: components.welo.identity.id, entry: { cid: entry.cid, @@ -183,27 +112,27 @@ describe('revisions', () => { } }]) - await sigint.interupt() + await components.stop() }) it('rpc - it exports a revision (file)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') + const { filesystem, components, socket } = await create() + const group = await createGroup(components, 'test') const fs = filesystem.getFileSystem(group) const path = '/test' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const sequence = 0 const dataFile = testData.data[0] const exportPath = dataFile.generatePath(testPath) assert(fs != null) - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + const [{ cid }] = await all(importer(components.helia.blockstore, dataFile.path)) const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, path, { @@ -220,7 +149,7 @@ describe('revisions', () => { const response = await client.rpc.request('export-revision', { group: group.toString(), path, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc'), + author: uint8ArrayToString(components.welo.identity.id, 'base58btc'), sequence, outPath: exportPath }) @@ -231,15 +160,15 @@ describe('revisions', () => { assert.equal(valid, true) - await sigint.interupt() + await components.stop() }) it('rpc - it exports a revision (directory)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') + const { filesystem, components, socket } = await create() + const group = await createGroup(components, 'test') const fs = filesystem.getFileSystem(group) const rootPath = '/test' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const sequence = 0 const outPath = Path.join(testPath, 'export-directory') @@ -248,12 +177,12 @@ describe('revisions', () => { for (const dataFile of testData.data) { const virtualPath = dataFile.generatePath(rootPath) - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + const [{ cid }] = await all(importer(components.helia.blockstore, dataFile.path)) const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, virtualPath, { @@ -271,7 +200,7 @@ describe('revisions', () => { const response = await client.rpc.request('export-revision', { group: group.toString(), path: rootPath, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc'), + author: uint8ArrayToString(components.welo.identity.id, 'base58btc'), sequence, outPath }) @@ -285,25 +214,25 @@ describe('revisions', () => { assert.equal(valid, true) } - await sigint.interupt() + await components.stop() }) it('rpc - lists a revision (file)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') + const { filesystem, components, socket } = await create() + const group = await createGroup(components, 'test') const fs = filesystem.getFileSystem(group) const path = '/test' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const dataFile = testData.data[0] assert(fs != null) - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + const [{ cid }] = await all(importer(components.helia.blockstore, dataFile.path)) const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) const before = Date.now() @@ -326,7 +255,7 @@ describe('revisions', () => { assert(Array.isArray(response)) assert.equal(response.length, 1) - assert.equal(response[0].author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(response[0].author, uint8ArrayToString(components.welo.identity.id, 'base58btc')) assert.equal(response[0].blocks, 1) assert.equal(response[0].cid, cid.toString()) assert.equal(response[0].encrypted, false) @@ -337,15 +266,15 @@ describe('revisions', () => { assert(response[0].timestamp >= before) assert(response[0].timestamp <= Date.now()) - await sigint.interupt() + await components.stop() }) it('rpc - it lists a revision (directory)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') + const { filesystem, components, socket } = await create() + const group = await createGroup(components, 'test') const fs = filesystem.getFileSystem(group) const rootPath = '/test' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) assert(fs != null) @@ -354,12 +283,12 @@ describe('revisions', () => { for (const dataFile of testData.data) { const virtualPath = dataFile.generatePath(rootPath) - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + const [{ cid }] = await all(importer(components.helia.blockstore, dataFile.path)) const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, virtualPath, { @@ -383,7 +312,7 @@ describe('revisions', () => { assert.equal(response.length, 3) for (const item of response) { - assert.equal(item.author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(item.author, uint8ArrayToString(components.welo.identity.id, 'base58btc')) assert.equal(item.blocks, 1) assert.equal(item.encrypted, false) assert.equal(item.priority, 1) @@ -401,14 +330,14 @@ describe('revisions', () => { assert(BigInt(item.size) === dataFile.size) } - await sigint.interupt() + await components.stop() }) it('rpc - read revision', async () => { - const { filesystem, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const ufs = unixfs(network.helia) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const ufs = unixfs(components.helia) const path = '/test' const data = 'test-data' @@ -417,7 +346,7 @@ describe('revisions', () => { const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, path, { @@ -435,7 +364,7 @@ describe('revisions', () => { group: group.toString(), path, sequence: 0, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc') + author: uint8ArrayToString(components.welo.identity.id, 'base58btc') } const read1 = await client.rpc.request('read', coreParams) @@ -455,6 +384,6 @@ describe('revisions', () => { assert.deepEqual(read4, data.slice(1, 3 + 1)) client.close() - await sigint.interupt() + await components.stop() }) }) diff --git a/packages/daemon/test/utils/create-group.ts b/packages/daemon/test/utils/create-group.ts index 063e7487..4aa4d24a 100644 --- a/packages/daemon/test/utils/create-group.ts +++ b/packages/daemon/test/utils/create-group.ts @@ -1,8 +1,7 @@ import { type CID } from 'multiformats/cid' -import type { Groups } from '../../src/common/groups.js' -import type { Welo } from 'welo' +import type { Components } from '../../src/common/interface.js' -export const createGroup = async (groups: Groups, welo: Welo, name: string, peers: Uint8Array[] = []): Promise => { +export const createGroup = async ({ welo, groups }: Components, name: string, peers: Uint8Array[] = []): Promise => { const manifest = await welo.determine({ name, meta: { type: 'group' }, From 1a7f87f1ef7139a7984677ffe1e7f8422b29d44d Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 16:30:53 +1300 Subject: [PATCH 50/62] Fix groups test. --- packages/daemon/test/modules/groups.spec.ts | 183 +++++++------------- 1 file changed, 61 insertions(+), 122 deletions(-) diff --git a/packages/daemon/test/modules/groups.spec.ts b/packages/daemon/test/modules/groups.spec.ts index f94d84f9..43bdaa6c 100644 --- a/packages/daemon/test/modules/groups.spec.ts +++ b/packages/daemon/test/modules/groups.spec.ts @@ -1,88 +1,20 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' -import { KeyManager, parseKeyData } from '@organicdesign/db-key-manager' import { createNetClient } from '@organicdesign/net-rpc' import * as cborg from 'cborg' import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' +import setup from '@/common/index.js' +import { createGroup } from '../utils/create-group.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { - Requires as GroupsComponents, - Provides as GroupsProvides -} from '../../src/modules/groups/index.js' +import type { Components } from '@/common/interface.js' describe('groups', () => { const testPath = mkTestPath('groups') - const mkGroup = async (m: GroupsProvides, name: string, peers: Uint8Array[] = []): Promise => { - const manifest = await m.welo.determine({ - name, - meta: { type: 'group' }, - access: { - protocol: '/hldb/access/static', - config: { write: [m.welo.identity.id, ...peers] } - } - }) - - await m.groups.add(manifest) - - return manifest.address.cid - } - - const create = async (name?: string): Promise & { - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: GroupsProvides - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - - const keyManager = name == null - ? undefined - : new KeyManager(parseKeyData({ - key: 'DpGbLiAX4wK4HHtG3DQb8cA6FG2ibv93X4ooZJ5LmMJJ-12FmenN8dbWysuYnzEHzmEF1hod4RGK8NfKFu1SEZ7XM', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n023330a98e30315e2233d4a31a6dc65d741a89f7ce6248e7de40c73995d23157' - })) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups - } - } - before(async () => { await fs.mkdir(testPath, { recursive: true }) }) @@ -91,25 +23,32 @@ describe('groups', () => { await fs.rm(testPath, { recursive: true }) }) + const create = async (): Promise<{ components: Components, socket: string }> => { + const socket = Path.join(testPath, `${Math.random()}.socket`) + const components = await setup({ socket }) + + return { components, socket } + } + it('creates a group', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') + const { components } = await create() + const group = await createGroup(components, 'test') assert(group) - await sigint.interupt() + await components.stop() }) it('tracker puts and checks a group\'s content', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) + const { components } = await create() + const group = await createGroup(components, 'test') + const database = components.groups.get(group) if (database == null) { throw new Error('group creation failed') } - const tracker = m.getTracker(database) + const tracker = components.getTracker(database) const key = 'test' const put = database.store.creators.put(key, 'my-data') @@ -122,19 +61,19 @@ describe('groups', () => { assert.equal(await tracker.validate(key, put), true) - await sigint.interupt() + await components.stop() }) it('tracker processes a group\'s content', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) + const { components } = await create() + const group = await createGroup(components, 'test') + const database = components.groups.get(group) if (database == null) { throw new Error('group creation failed') } - const tracker = m.getTracker(database) + const tracker = components.getTracker(database) const key = '/test' const value = 'my-data' const put = database.store.creators.put(key, value) @@ -149,19 +88,19 @@ describe('groups', () => { entries = await all(tracker.process()) assert.deepEqual(entries, []) - await sigint.interupt() + await components.stop() }) it('tracker is scope limited', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) + const { components } = await create() + const group = await createGroup(components, 'test') + const database = components.groups.get(group) if (database == null) { throw new Error('group creation failed') } - const tracker = m.getTracker(database) + const tracker = components.getTracker(database) const key = '/test' const value = 'my-data' const put = database.store.creators.put(key, value) @@ -174,110 +113,110 @@ describe('groups', () => { entries = await all(tracker.process(key)) assert.deepEqual(entries, [{ key, value: cborg.encode(value) }]) - await sigint.interupt() + await components.stop() }) it('uses the identity from base in welo', async () => { - const { groups: m, sigint, base } = await create() + const { components } = await create() - assert.deepEqual(m.welo.identity, await base.keyManager.getWeloIdentity()) + assert.deepEqual(components.welo.identity, await components.keyManager.getWeloIdentity()) - await sigint.interupt() + await components.stop() }) it('rpc - id returns the base58btc formatted welo id', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const id = await client.rpc.request('id', {}) - assert.equal(uint8ArrayToString(m.welo.identity.id, 'base58btc'), id) + assert.equal(uint8ArrayToString(components.welo.identity.id, 'base58btc'), id) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - create groups creates a group without other peers', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const name = 'test' const cid = await client.rpc.request('create-group', { name, peers: [] }) const group = CID.parse(cid) - const database = m.groups.get(group) + const database = components.groups.get(group) assert(database != null) assert.equal(database.manifest.name, name) - assert.deepEqual(database.manifest.access.config?.write, [m.welo.identity.id]) + assert.deepEqual(database.manifest.access.config?.write, [components.welo.identity.id]) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - create groups creates a group with other peers', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const name = 'test' const otherPeer = 'GZsJqUjmbVqZCUMbJoe5ye4xfdKZVPVwBoFFQiyCZYesq6Us5b' const cid = await client.rpc.request('create-group', { name, peers: [otherPeer] }) const group = CID.parse(cid) - const database = m.groups.get(group) + const database = components.groups.get(group) assert(database != null) assert.equal(database.manifest.name, name) assert.deepEqual(database.manifest.access.config?.write, [ - m.welo.identity.id, + components.welo.identity.id, uint8ArrayFromString(otherPeer, 'base58btc') ]) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - joins an external group', async () => { - const components = await Promise.all([create(), create('server-join-group')]) - const client = createNetClient(components[0].argv.socket) + const components = await Promise.all([create(), create()]) + const client = createNetClient(components[0].socket) const name = 'test' - const group = await mkGroup(components[1].groups, name) + const group = await createGroup(components[1].components, name) - await components[0].network.libp2p.dial(components[1].network.libp2p.getMultiaddrs()) + await components[0].components.libp2p.dial(components[1].components.libp2p.getMultiaddrs()) const res = await client.rpc.request('join-group', { group: group.toString() }) assert.equal(res, null) - const database = components[0].groups.groups.get(group) + const database = components[0].components.groups.get(group) assert(database) assert.equal(database.manifest.name, name) client.close() - await Promise.all(components.map(async c => c.sigint.interupt())) + await Promise.all(components.map(async c => c.components.stop())) }) it('rpc - list groups', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const name = 'test' let groups = await client.rpc.request('list-groups', {}) assert.deepEqual(groups, []) - const group = await mkGroup(m, name) + const group = await createGroup(components, name) groups = await client.rpc.request('list-groups', {}) assert.deepEqual(groups, [{ group: group.toString(), name }]) client.close() - await sigint.interupt() + await components.stop() }) // This fails it github too - seems to think the `server-sync-groups` socket is in use? it.skip('rpc - sync groups', async () => { - const components = await Promise.all([create(), create('server-sync-groups')]) - const client = createNetClient(components[0].argv.socket) + const components = await Promise.all([create(), create()]) + const client = createNetClient(components[0].socket) const key = '/test' const value = 'test-value' @@ -285,8 +224,8 @@ describe('groups', () => { assert.deepEqual(groups, []) - const group = await mkGroup(components[1].groups, 'test') - const database = components[1].groups.groups.get(group) + const group = await createGroup(components[1].components, 'test') + const database = components[1].components.groups.get(group) if (database == null) { throw new Error('database creation failed') @@ -296,7 +235,7 @@ describe('groups', () => { await database.replica.write(put) - await components[0].network.libp2p.dial(components[1].network.libp2p.getMultiaddrs()) + await components[0].components.libp2p.dial(components[1].components.libp2p.getMultiaddrs()) await client.rpc.request('join-group', { group: group.toString() }) await client.rpc.request('sync', {}) @@ -306,6 +245,6 @@ describe('groups', () => { assert.deepEqual(result, value) client.close() - await Promise.all(components.map(async c => c.sigint.interupt())) + await Promise.all(components.map(async c => c.components.stop())) }) }) From 4fce7401a05e72c4bac543144255a8d9eb7d16de Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 16:42:16 +1300 Subject: [PATCH 51/62] Fix filesystem test. --- .../daemon/test/modules/filesystem.spec.ts | 286 +++++++----------- .../daemon/test/modules/revisions.spec.ts | 4 +- 2 files changed, 114 insertions(+), 176 deletions(-) diff --git a/packages/daemon/test/modules/filesystem.spec.ts b/packages/daemon/test/modules/filesystem.spec.ts index 2c1d8896..58bd67eb 100644 --- a/packages/daemon/test/modules/filesystem.spec.ts +++ b/packages/daemon/test/modules/filesystem.spec.ts @@ -2,7 +2,6 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' import { unixfs } from '@helia/unixfs' -import { KeyManager } from '@organicdesign/db-key-manager' import * as testData from '@organicdesign/db-test-utils' import { importer } from '@organicdesign/db-utils' import { createNetClient } from '@organicdesign/net-rpc' @@ -11,88 +10,27 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createDownloader from '../../src/modules/downloader/index.js' -import createFilesystem from '../../src/modules/filesystem/index.js' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import createTick from '../../src/modules/tick/index.js' import { createGroup } from '../utils/create-group.js' import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { Provides as FSProvides } from '../../src/modules/filesystem/index.js' +import type { Context as FilesystemContext } from '../../src/modules/filesystem/index.js' +import type { Components } from '@/common/interface.js' +import setupFilesystem from '../../src/modules/filesystem/index.js' +import setup from '@/common/index.js' describe('filesystem', () => { const testPath = mkTestPath('filesystem') - const create = async (name?: string): Promise<{ - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: Awaited> - filesystem: FSProvides - sigint: Awaited> - tick: Awaited> - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) + const create = async (): Promise<{ + filesystem: FilesystemContext + components: Components + socket: string + }> => { + const socket = Path.join(testPath, `${Math.random()}.socket`) + const components = await setup({ socket }) + const filesystem = await setupFilesystem(components) - const tick = await createTick({ config, sigint }) - - const filesystem = await createFilesystem({ - sigint, - base, - rpc, - network, - groups, - downloader, - tick, - config - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups, - filesystem, - tick - } + return { filesystem, components, socket } } before(async () => { @@ -104,44 +42,44 @@ describe('filesystem', () => { }) it('uses all as the default revision strategy', async () => { - const { filesystem: m, sigint } = await create() + const { filesystem, components } = await create() - assert.equal(m.config.defaultRevisionStrategy, 'all') + assert.equal(filesystem.config.defaultRevisionStrategy, 'all') - await sigint.interupt() + await components.stop() }) it('returns null when wraping a group that doesn\'t exist', async () => { - const { filesystem: m, sigint } = await create() - const fs = m.getFileSystem(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) + const { filesystem, components } = await create() + const fs = filesystem.getFileSystem(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) assert.equal(fs, null) - await sigint.interupt() + await components.stop() }) it('wraps a group in a filesystem', async () => { - const { filesystem: m, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components } = await create() + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) assert.notEqual(fs, null) - await sigint.interupt() + await components.stop() }) it('uploads show in filesystem', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components } = await create() + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const path = '/test' assert(fs != null) const before = Date.now() - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -149,11 +87,11 @@ describe('filesystem', () => { }]) const entry = await fs.get(path) - const values = await Promise.all(dag.map(async d => network.helia.blockstore.get(d))) + const values = await Promise.all(dag.map(async d => components.helia.blockstore.get(d))) const size = values.reduce((a, c) => a + c.length, 0) assert(entry != null) - assert.deepEqual(entry.author, groups.welo.identity.id) + assert.deepEqual(entry.author, components.welo.identity.id) assert.equal(entry.blocks, dag.length) assert.deepEqual(entry.cid, dag[0]) assert.equal(entry.encrypted, false) @@ -164,22 +102,22 @@ describe('filesystem', () => { assert(entry.timestamp >= before) assert(entry.timestamp <= Date.now()) - await sigint.interupt() + await components.stop() }) it('emits the file:added event when the FS was written to', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components } = await create() + const group = await createGroup(components, 'test') + const dag = await createDag(components.helia, 2, 2) const path = '/test' const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - m.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -188,26 +126,26 @@ describe('filesystem', () => { await promise - await sigint.interupt() + await components.stop() }) it('local settings change FS output', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components } = await create() + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const path = '/test' assert(fs != null) - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, priority: 1 }]) - await m.localSettings.set(group, path, { + await filesystem.localSettings.set(group, path, { priority: 100 }) @@ -216,20 +154,20 @@ describe('filesystem', () => { assert(entry != null) assert.equal(entry.priority, 100) - await sigint.interupt() + await components.stop() }) it('rpc - delete (file)', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const path = '/test' assert(fs != null) - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -245,22 +183,22 @@ describe('filesystem', () => { assert.equal(entry, null) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - delete (directory)', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const rootPath = '/test' const paths = [`${rootPath}/file1`, `${rootPath}/file2`, `${rootPath}/sub/file3`] assert(fs != null) await Promise.all(paths.map(async path => - m.uploads.add('put', [group.bytes, path, { + filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -277,14 +215,14 @@ describe('filesystem', () => { assert.deepEqual(entries, []) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - edit priority', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const path = '/test' const priority = 50 @@ -294,28 +232,28 @@ describe('filesystem', () => { assert.equal(response, null) - const localSettings = await m.localSettings.get(group, path) + const localSettings = await filesystem.localSettings.get(group, path) assert.equal(localSettings.priority, priority) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - export (file)', async () => { - const { filesystem: m, network, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const rootPath = '/test' const outPath = Path.join(testPath, 'export-file') assert(fs != null) await Promise.all(testData.data.map(async data => { - const result = await all(importer(network.helia.blockstore, data.path)) + const result = await all(importer(components.helia.blockstore, data.path)) - await m.uploads.add('put', [group.bytes, data.generatePath(rootPath), { + await filesystem.uploads.add('put', [group.bytes, data.generatePath(rootPath), { cid: result[0].cid.bytes, encrypted: false, revisionStrategy: 'all', @@ -339,23 +277,23 @@ describe('filesystem', () => { } client.close() - await sigint.interupt() + await components.stop() }) it('rpc - export (directory)', async () => { - const { filesystem: m, network, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const rootPath = '/test' const outPath = Path.join(testPath, 'export-directory') assert(fs != null) await Promise.all(testData.data.map(async data => { - const result = await all(importer(network.helia.blockstore, data.path)) + const result = await all(importer(components.helia.blockstore, data.path)) - await m.uploads.add('put', [group.bytes, data.generatePath(rootPath), { + await filesystem.uploads.add('put', [group.bytes, data.generatePath(rootPath), { cid: result[0].cid.bytes, encrypted: false, revisionStrategy: 'all', @@ -379,14 +317,14 @@ describe('filesystem', () => { } client.close() - await sigint.interupt() + await components.stop() }) it('rpc - import (file)', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const rootPath = '/test' assert(fs != null) @@ -413,14 +351,14 @@ describe('filesystem', () => { })) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - import (directory)', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const rootPath = '/test' assert(fs != null) @@ -456,15 +394,15 @@ describe('filesystem', () => { } client.close() - await sigint.interupt() + await components.stop() }) it('rpc - list', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const rootPath = '/test' const paths = [`${rootPath}/file1`, `${rootPath}/file2`, `${rootPath}/sub/file3`] @@ -473,7 +411,7 @@ describe('filesystem', () => { const before = Date.now() await Promise.all(paths.map(async path => - m.uploads.add('put', [group.bytes, path, { + filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -485,12 +423,12 @@ describe('filesystem', () => { assert(Array.isArray(response)) - const values = await Promise.all(dag.map(async d => network.helia.blockstore.get(d))) + const values = await Promise.all(dag.map(async d => components.helia.blockstore.get(d))) const size = values.reduce((a, c) => a + c.length, 0) for (const entry of response) { assert(entry != null) - assert.equal(entry.author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(entry.author, uint8ArrayToString(components.welo.identity.id, 'base58btc')) assert.equal(entry.blocks, dag.length) assert.deepEqual(entry.cid, dag[0].toString()) assert.equal(entry.encrypted, false) @@ -502,20 +440,20 @@ describe('filesystem', () => { } client.close() - await sigint.interupt() + await components.stop() }) it('rpc - read', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const ufs = unixfs(network.helia) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const ufs = unixfs(components.helia) const path = '/test' const data = 'test-data' const cid = await ufs.addBytes(uint8ArrayFromString(data)) - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: cid.bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -539,14 +477,14 @@ describe('filesystem', () => { assert.deepEqual(read4, data.slice(1, 3 + 1)) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - write', async () => { - const { filesystem: m, base, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const ufs = unixfs({ blockstore: new MemoryBlockstore() }) const path = '/test' const data = 'test-data' @@ -563,7 +501,7 @@ describe('filesystem', () => { const entry1 = await fs.get(path) assert(entry1 != null) - assert.deepEqual(entry1.author, groups.welo.identity.id) + assert.deepEqual(entry1.author, components.welo.identity.id) assert.equal(entry1.blocks, 1) assert.deepEqual(entry1.cid, cid) assert.equal(entry1.encrypted, false) @@ -582,7 +520,7 @@ describe('filesystem', () => { assert(entry2 != null) - const value2 = await base.blockstore.get(entry2.cid) + const value2 = await components.blockstore.get(entry2.cid) assert.deepEqual(value2, uint8ArrayFromString(newData2)) @@ -595,7 +533,7 @@ describe('filesystem', () => { assert(entry3 != null) - const value3 = await base.blockstore.get(entry3.cid) + const value3 = await components.blockstore.get(entry3.cid) assert.deepEqual(value3, uint8ArrayFromString('test-data-long')) @@ -608,11 +546,11 @@ describe('filesystem', () => { assert(entry4 != null) - const value4 = await base.blockstore.get(entry4.cid) + const value4 = await components.blockstore.get(entry4.cid) assert.deepEqual(value4, uint8ArrayFromString('test-long-long')) client.close() - await sigint.interupt() + await components.stop() }) }) diff --git a/packages/daemon/test/modules/revisions.spec.ts b/packages/daemon/test/modules/revisions.spec.ts index 9f95ccdb..fb23c95d 100644 --- a/packages/daemon/test/modules/revisions.spec.ts +++ b/packages/daemon/test/modules/revisions.spec.ts @@ -22,13 +22,13 @@ import { createDag } from '../utils/dag.js' describe('revisions', () => { const testPath = mkTestPath('revisions') - const create = async (name?: string): Promise<{ + const create = async (): Promise<{ revisions: RevisionsContext filesystem: FilesystemContext components: Components socket: string }> => { - const socket = Path.join(testPath, `${name ?? 'server'}.socket`) + const socket = Path.join(testPath, `${Math.random()}.socket`) const components = await setup({ socket }) const filesystem = await setupFilesystem(components) const revisions = await setupRevisions(components) From ac805ffe9fd548731fc8f7cb2fa31b72f039a596 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 16:49:42 +1300 Subject: [PATCH 52/62] Fix download test. --- .../daemon/test/modules/downloader.spec.ts | 99 +++++-------------- 1 file changed, 24 insertions(+), 75 deletions(-) diff --git a/packages/daemon/test/modules/downloader.spec.ts b/packages/daemon/test/modules/downloader.spec.ts index 5c459870..f7a5ba32 100644 --- a/packages/daemon/test/modules/downloader.spec.ts +++ b/packages/daemon/test/modules/downloader.spec.ts @@ -1,65 +1,22 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' -import { KeyManager } from '@organicdesign/db-key-manager' import { createNetClient } from '@organicdesign/net-rpc' import { MemoryBlockstore } from 'blockstore-core' import { CID } from 'multiformats/cid' -import createDownloader from '../../src/modules/downloader/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { - Requires as DownloaderComponents, - Provides as DownloaderProvides -} from '../../src/modules/downloader/index.js' +import type { Components} from '@/common/interface.js' +import setup from '@/common/index.js' describe('downloader', () => { const testPath = mkTestPath('groups') - const create = async (name?: string): Promise & { - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - downloader: DownloaderProvides - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) + const create = async (): Promise<{ components: Components, socket: string }> => { + const socket = Path.join(testPath, `${Math.random()}.socket`) + const components = await setup({ socket }) - return { - argv, - config, - sigint, - rpc, - base, - network, - downloader - } + return { components, socket } } before(async () => { @@ -70,48 +27,40 @@ describe('downloader', () => { await fs.rm(testPath, { recursive: true }) }) - it('has 20 slots by default', async () => { - const { downloader: m, sigint } = await create() - - assert.equal(m.config.slots, 20) - - await sigint.interupt() - }) - it('rpc - set priority updates local priority', async () => { const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' const path = '/test.txt' const priority = 50 - const { downloader: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const key = Path.join('/', group, path) - await m.pinManager.put(key, { priority: 1, cid: CID.parse(group) }) + await components.pinManager.put(key, { priority: 1, cid: CID.parse(group) }) const response = await client.rpc.request('set-priority', { group, path, priority }) assert.equal(response, null) - const pinData = await m.pinManager.get(key) + const pinData = await components.pinManager.get(key) assert(pinData != null) assert.equal(pinData.priority, priority) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - get speed', async () => { - const { downloader: m, network, sigint, argv } = await create() + const { components, socket } = await create() const blockstore = new MemoryBlockstore() const dag = await createDag({ blockstore }, 2, 2) const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' const path = '/test.txt' const range = 500 - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const key = Path.join('/', group, path) - await m.pinManager.put(key, { priority: 1, cid: dag[0] }) + await components.pinManager.put(key, { priority: 1, cid: dag[0] }) const speed1 = await client.rpc.request('get-speeds', { cids: [dag[0].toString()], @@ -122,7 +71,7 @@ describe('downloader', () => { const value = await blockstore.get(dag[0]) - await network.helia.blockstore.put(dag[0], value) + await components.helia.blockstore.put(dag[0], value) await new Promise(resolve => setTimeout(resolve, range / 2)) const speed2 = await client.rpc.request('get-speeds', { @@ -140,8 +89,8 @@ describe('downloader', () => { ]) await Promise.all([ - network.helia.blockstore.put(dag[1], values[0]), - network.helia.blockstore.put(dag[4], values[1]) + components.helia.blockstore.put(dag[1], values[0]), + components.helia.blockstore.put(dag[4], values[1]) ]) await new Promise(resolve => setTimeout(resolve, range / 2)) @@ -157,16 +106,16 @@ describe('downloader', () => { }]) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - get status', async () => { - const { downloader: m, network, sigint, argv } = await create() + const { components, socket } = await create() const blockstore = new MemoryBlockstore() const dag = await createDag({ blockstore }, 2, 2) const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' const path = '/test.txt' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const key = Path.join('/', group, path) const status1 = await client.rpc.request('get-status', { @@ -180,7 +129,7 @@ describe('downloader', () => { state: 'NOTFOUND' }]) - await m.pinManager.put(key, { priority: 1, cid: dag[0] }) + await components.pinManager.put(key, { priority: 1, cid: dag[0] }) const status2 = await client.rpc.request('get-status', { cids: [dag[0].toString()] @@ -195,7 +144,7 @@ describe('downloader', () => { const value = await blockstore.get(dag[0]) - await network.helia.blockstore.put(dag[0], value) + await components.helia.blockstore.put(dag[0], value) await new Promise(resolve => setTimeout(resolve, 100)) const status3 = await client.rpc.request('get-status', { @@ -212,7 +161,7 @@ describe('downloader', () => { const values = await Promise.all(dag.map(async cid => { const value = await blockstore.get(cid) - await network.helia.blockstore.put(cid, value) + await components.helia.blockstore.put(cid, value) return value.length })) @@ -231,6 +180,6 @@ describe('downloader', () => { }]) client.close() - await sigint.interupt() + await components.stop() }) }) From d721d8a4d5d00cd0064468ffc385628f42d8acc7 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 17:01:42 +1300 Subject: [PATCH 53/62] Fix base test. --- packages/daemon/test/modules/base.spec.ts | 76 ++++++++++------------- 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/packages/daemon/test/modules/base.spec.ts b/packages/daemon/test/modules/base.spec.ts index 81f9c180..11a9aaea 100644 --- a/packages/daemon/test/modules/base.spec.ts +++ b/packages/daemon/test/modules/base.spec.ts @@ -8,26 +8,21 @@ import { FsDatastore } from 'datastore-fs' import { Key } from 'interface-datastore' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays' -import base from '../../src/modules/base/index.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockConfig from './mock-config.js' -import type { Provides as Argv } from '../../src/modules/argv/index.js' import { extendDatastore } from '@/utils.js' +import setup from '@/common/index.js' const parseStr = (data: string): Uint8Array => uint8ArrayFromString(data, 'base64') const testPath = mkTestPath('base') describe('base', () => { - let argv: Argv + const keyPath = Path.join(testPath, 'key.json') + const socket = Path.join(testPath, 'server.socket') before(async () => { - argv = mockArgv(testPath) - - await fs.mkdir(Path.join(argv.key, '..'), { recursive: true }) await fs.mkdir(testPath, { recursive: true }) - await fs.writeFile(argv.key, JSON.stringify({ + await fs.writeFile(keyPath, JSON.stringify({ key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' })) @@ -38,67 +33,61 @@ describe('base', () => { }) it('returns the key manager', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) + const components = await setup({ key: keyPath, socket }) assert.deepEqual( - new Uint8Array(m.keyManager.aesKey), + new Uint8Array(components.keyManager.aesKey), parseStr('knUGn6uUeQcoxfM1qAtg3F/Njm4bp+GcZK257NZ5AtE') ) assert.deepEqual( - new Uint8Array(m.keyManager.hmacKey), + new Uint8Array(components.keyManager.hmacKey), parseStr('KZkJfNz3bRrn6XHvWYtD4+dXvmhdT4TBhBIkWn8y3jY') ) assert.deepEqual( - (await m.keyManager.getWeloIdentity()).id, + (await components.keyManager.getWeloIdentity()).id, parseStr('CAISIQPSvjmKINqJY5SA/3c+kadFmIsHeTXtTJYlrooZ53DTUg') ) assert.deepEqual( - (await m.keyManager.getPeerId()).toBytes(), + (await components.keyManager.getPeerId()).toBytes(), parseStr('ACUIAhIhA5m2/DfXxqi0i+fyYixRaWGirDEVemxUEv8WMZPwFPZB') ) assert.deepEqual( - m.keyManager.getPskKey(), + components.keyManager.getPskKey(), parseStr('L2tleS9zd2FybS9wc2svMS4wLjAvCi9iYXNlMTYvCjU2ZDNjMTgyODJmMWYxYjFiM2UwNGU0MGRkNWQ4YmY0NGNhZmE4YmM5YzliYzdjNTc3MTZhNzc2NmZhMmM1NTA') ) + + await components.stop() }) it('uses memory blockstore when memory is specified', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) + const components = await setup({ socket, config: { storage: ':memory:' } }) - assert(m.blockstore instanceof MemoryBlockstore) + assert(components.blockstore instanceof MemoryBlockstore) + + await components.stop() }) it('uses memory datastore when memory is specified', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) + const components = await setup({ socket, config: { storage: ':memory:' } }) + + assert(components.datastore instanceof MemoryDatastore) - assert(m.datastore instanceof MemoryDatastore) + await components.stop() }) it('uses fs blockstore when a path is specified', async () => { const blockstorePath = Path.join(testPath, 'blockstore') const testData = uint8ArrayFromString('test') - const m = await base({ - config: mockConfig({ storage: testPath }), - argv - }) + const components = await setup({ socket, config: { storage: testPath } }) - assert(m.blockstore instanceof FsBlockstore) + assert(components.blockstore instanceof FsBlockstore) - await m.blockstore.put(CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ'), testData) + await components.blockstore.put(CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ'), testData) const out = await fs.readdir(blockstorePath, { recursive: true }) @@ -110,21 +99,22 @@ describe('base', () => { const blockData = await fs.readFile(Path.join(blockstorePath, '7I/BCIQLASSX2QHMUE4IBHYTTJ3LCICEGM6DOQBZDSN7DTU5RYQ2PEQQX7I.data')) assert.deepEqual(new Uint8Array(blockData), testData) + + await components.stop() }) it('uses fs datastore when a path is specified', async () => { - const datastorePath = Path.join(testPath, 'datastore') + const datastorePath = Path.join(testPath, 'datastore', 'test') + + const components = await setup({ socket, config: { storage: testPath } }) - const m = await base({ - config: mockConfig({ storage: testPath }), - argv - }) + assert(components.datastore instanceof FsDatastore) - assert(m.datastore instanceof FsDatastore) + const datastore = extendDatastore(components.datastore, 'test') - await m.datastore.put(new Key('key'), uint8ArrayFromString('value')) + await datastore.put(new Key('key'), uint8ArrayFromString('value')) - const subDatastore = extendDatastore(extendDatastore(m.datastore, 'a'), 'b/c') + const subDatastore = extendDatastore(extendDatastore(datastore, 'a'), 'b/c') await subDatastore.put(new Key('d/e'), uint8ArrayFromString('test')) const out = await fs.readdir(datastorePath, { recursive: true }) @@ -143,5 +133,7 @@ describe('base', () => { assert.deepEqual(new Uint8Array(data1), uint8ArrayFromString('value')) assert.deepEqual(new Uint8Array(data2), uint8ArrayFromString('test')) + + await components.stop() }) }) From 24275509cc5a2d931b9669147d0422436b40fd9d Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 17:02:23 +1300 Subject: [PATCH 54/62] Delete old mocks. --- packages/daemon/test/modules/mock-argv.ts | 12 ------------ packages/daemon/test/modules/mock-base.ts | 20 -------------------- packages/daemon/test/modules/mock-config.ts | 12 ------------ 3 files changed, 44 deletions(-) delete mode 100644 packages/daemon/test/modules/mock-argv.ts delete mode 100644 packages/daemon/test/modules/mock-base.ts delete mode 100644 packages/daemon/test/modules/mock-config.ts diff --git a/packages/daemon/test/modules/mock-argv.ts b/packages/daemon/test/modules/mock-argv.ts deleted file mode 100644 index c6da9904..00000000 --- a/packages/daemon/test/modules/mock-argv.ts +++ /dev/null @@ -1,12 +0,0 @@ -import Path from 'path' -import type { Provides } from '../../src/modules/argv/index.js' - -const provs = (path: string): Provides => { - return { - key: Path.join(path, 'key.json'), - config: Path.join(path, 'config.json'), - socket: Path.join(path, 'server.socket') - } -} - -export default provs diff --git a/packages/daemon/test/modules/mock-base.ts b/packages/daemon/test/modules/mock-base.ts deleted file mode 100644 index a139b417..00000000 --- a/packages/daemon/test/modules/mock-base.ts +++ /dev/null @@ -1,20 +0,0 @@ -import Path from 'path' -import { KeyManager, parseKeyData } from '@organicdesign/db-key-manager' -import { MemoryBlockstore } from 'blockstore-core' -import { FsBlockstore } from 'blockstore-fs' -import { MemoryDatastore } from 'datastore-core' -import { FsDatastore } from 'datastore-fs' -import type { Provides } from '../../src/modules/base/index.js' - -const provs = (config?: { path?: string, keyManager?: KeyManager }): Provides => { - return { - keyManager: config?.keyManager ?? new KeyManager(parseKeyData({ - key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' - })), - datastore: config?.path == null ? new MemoryDatastore() : new FsDatastore(Path.join(config?.path, 'datastore')), - blockstore: config?.path == null ? new MemoryBlockstore() : new FsBlockstore(Path.join(config?.path, 'blockstore')) - } -} - -export default provs diff --git a/packages/daemon/test/modules/mock-config.ts b/packages/daemon/test/modules/mock-config.ts deleted file mode 100644 index 45bf3666..00000000 --- a/packages/daemon/test/modules/mock-config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { type z } from 'zod' -import type { Provides } from '../../src/modules/config/index.js' - -const provs = (config: Record): Provides => ({ - config, - - get (schema: T): z.infer { - return schema.parse(config) - } -}) - -export default provs From e69ba8877716f9e7e2602e2aee98601bb34d3774 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 17:07:48 +1300 Subject: [PATCH 55/62] Delete old sigint test. --- packages/daemon/test/modules/sigint.spec.ts | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 packages/daemon/test/modules/sigint.spec.ts diff --git a/packages/daemon/test/modules/sigint.spec.ts b/packages/daemon/test/modules/sigint.spec.ts deleted file mode 100644 index 2aa2c68a..00000000 --- a/packages/daemon/test/modules/sigint.spec.ts +++ /dev/null @@ -1,15 +0,0 @@ -import assert from 'assert/strict' -import sigint from '../../src/modules/sigint/index.js' - -describe('sigint', () => { - it('calls the interupt method when interupt gets called', async () => { - const m = await sigint() - - let called = 0 - - m.onInterupt(() => called++) - await m.interupt() - - assert.equal(called, 1) - }) -}) From 3703d6d1ba7b471deb44d1065e3f20bd1db4743f Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 17:08:37 +1300 Subject: [PATCH 56/62] Fix rpc test. --- packages/daemon/test/modules/rpc.spec.ts | 30 ++++++------------------ 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/packages/daemon/test/modules/rpc.spec.ts b/packages/daemon/test/modules/rpc.spec.ts index da8324cc..6623de46 100644 --- a/packages/daemon/test/modules/rpc.spec.ts +++ b/packages/daemon/test/modules/rpc.spec.ts @@ -1,51 +1,34 @@ import assert from 'assert/strict' import fs from 'fs/promises' import { createNetClient } from '@organicdesign/net-rpc' -import rpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import type { Provides as Argv } from '../../src/modules/argv/index.js' -import type { Provides as Sigint } from '../../src/modules/sigint/index.js' +import setup from '@/common/index.js' +import Path from 'path' const testPath = mkTestPath('rpc') describe('rpc', () => { - let argv: Argv - let sigint: Sigint + const socket = Path.join(testPath, 'server.socket') before(async () => { - argv = mockArgv(testPath) - sigint = await createSigint() - await fs.mkdir(testPath, { recursive: true }) - - await fs.writeFile(argv.key, JSON.stringify({ - key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' - })) }) after(async () => { await fs.rm(testPath, { recursive: true }) - await sigint.interupt() }) it('adds RPC methods', async () => { + const components = await setup({ socket }) const testData = { key: 'value' } const returnData = { return: 'return-value' } - const m = await rpc({ - argv, - sigint - }) - - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const methodPromise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error()) }, 50) - m.addMethod('test', async params => { + components.net.rpc.addMethod('test', async params => { resolve(params) return returnData }) @@ -57,5 +40,6 @@ describe('rpc', () => { assert.deepEqual(await methodPromise, testData) client.close() + await components.stop() }) }) From c7d9ce9538fb24e18404387444743e2eb39f4041 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 17:12:17 +1300 Subject: [PATCH 57/62] Linting. --- packages/daemon/test/modules/base.spec.ts | 22 ++++++------- packages/daemon/test/modules/config.spec.ts | 10 +++--- .../daemon/test/modules/downloader.spec.ts | 2 +- .../daemon/test/modules/filesystem.spec.ts | 18 +++++------ packages/daemon/test/modules/groups.spec.ts | 12 +++---- packages/daemon/test/modules/network.spec.ts | 12 +++---- .../daemon/test/modules/revisions.spec.ts | 32 +++++++++---------- packages/daemon/test/modules/rpc.spec.ts | 6 ++-- 8 files changed, 57 insertions(+), 57 deletions(-) diff --git a/packages/daemon/test/modules/base.spec.ts b/packages/daemon/test/modules/base.spec.ts index 11a9aaea..dbccc8b5 100644 --- a/packages/daemon/test/modules/base.spec.ts +++ b/packages/daemon/test/modules/base.spec.ts @@ -9,15 +9,15 @@ import { Key } from 'interface-datastore' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays' import { mkTestPath } from '../utils/paths.js' -import { extendDatastore } from '@/utils.js' import setup from '@/common/index.js' +import { extendDatastore } from '@/utils.js' const parseStr = (data: string): Uint8Array => uint8ArrayFromString(data, 'base64') const testPath = mkTestPath('base') describe('base', () => { - const keyPath = Path.join(testPath, 'key.json') - const socket = Path.join(testPath, 'server.socket') + const keyPath = Path.join(testPath, 'key.json') + const socket = Path.join(testPath, 'server.socket') before(async () => { await fs.mkdir(testPath, { recursive: true }) @@ -60,7 +60,7 @@ describe('base', () => { parseStr('L2tleS9zd2FybS9wc2svMS4wLjAvCi9iYXNlMTYvCjU2ZDNjMTgyODJmMWYxYjFiM2UwNGU0MGRkNWQ4YmY0NGNhZmE4YmM5YzliYzdjNTc3MTZhNzc2NmZhMmM1NTA') ) - await components.stop() + await components.stop() }) it('uses memory blockstore when memory is specified', async () => { @@ -68,7 +68,7 @@ describe('base', () => { assert(components.blockstore instanceof MemoryBlockstore) - await components.stop() + await components.stop() }) it('uses memory datastore when memory is specified', async () => { @@ -76,14 +76,14 @@ describe('base', () => { assert(components.datastore instanceof MemoryDatastore) - await components.stop() + await components.stop() }) it('uses fs blockstore when a path is specified', async () => { const blockstorePath = Path.join(testPath, 'blockstore') const testData = uint8ArrayFromString('test') - const components = await setup({ socket, config: { storage: testPath } }) + const components = await setup({ socket, config: { storage: testPath } }) assert(components.blockstore instanceof FsBlockstore) @@ -100,17 +100,17 @@ describe('base', () => { assert.deepEqual(new Uint8Array(blockData), testData) - await components.stop() + await components.stop() }) it('uses fs datastore when a path is specified', async () => { const datastorePath = Path.join(testPath, 'datastore', 'test') - const components = await setup({ socket, config: { storage: testPath } }) + const components = await setup({ socket, config: { storage: testPath } }) assert(components.datastore instanceof FsDatastore) - const datastore = extendDatastore(components.datastore, 'test') + const datastore = extendDatastore(components.datastore, 'test') await datastore.put(new Key('key'), uint8ArrayFromString('value')) @@ -134,6 +134,6 @@ describe('base', () => { assert.deepEqual(new Uint8Array(data1), uint8ArrayFromString('value')) assert.deepEqual(new Uint8Array(data2), uint8ArrayFromString('test')) - await components.stop() + await components.stop() }) }) diff --git a/packages/daemon/test/modules/config.spec.ts b/packages/daemon/test/modules/config.spec.ts index c47bc0b8..df1fcb81 100644 --- a/packages/daemon/test/modules/config.spec.ts +++ b/packages/daemon/test/modules/config.spec.ts @@ -30,13 +30,13 @@ after(async () => { describe('config', () => { it('gets config from file', async () => { - const config = await parseConfig(configPath) + const config = await parseConfig(configPath) assert.deepEqual(config, configData) }) - it('parses config from schema', async () => { - const components = await setup({ config: configData, socket: Path.join(testPath, 'server.socket') }) + it('parses config from schema', async () => { + const components = await setup({ config: configData, socket: Path.join(testPath, 'server.socket') }) assert.deepEqual( components.parseConfig(z.object({ bootstrap: z.array(z.string()) })), @@ -48,6 +48,6 @@ describe('config', () => { { tickInterval: configData.tickInterval, private: configData.private } ) - await components.stop() - }) + await components.stop() + }) }) diff --git a/packages/daemon/test/modules/downloader.spec.ts b/packages/daemon/test/modules/downloader.spec.ts index f7a5ba32..fa5c4246 100644 --- a/packages/daemon/test/modules/downloader.spec.ts +++ b/packages/daemon/test/modules/downloader.spec.ts @@ -6,7 +6,7 @@ import { MemoryBlockstore } from 'blockstore-core' import { CID } from 'multiformats/cid' import { createDag } from '../utils/dag.js' import { mkTestPath } from '../utils/paths.js' -import type { Components} from '@/common/interface.js' +import type { Components } from '@/common/interface.js' import setup from '@/common/index.js' describe('downloader', () => { diff --git a/packages/daemon/test/modules/filesystem.spec.ts b/packages/daemon/test/modules/filesystem.spec.ts index 58bd67eb..18a58032 100644 --- a/packages/daemon/test/modules/filesystem.spec.ts +++ b/packages/daemon/test/modules/filesystem.spec.ts @@ -10,27 +10,27 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import setupFilesystem from '../../src/modules/filesystem/index.js' import { createGroup } from '../utils/create-group.js' import { createDag } from '../utils/dag.js' import { mkTestPath } from '../utils/paths.js' import type { Context as FilesystemContext } from '../../src/modules/filesystem/index.js' import type { Components } from '@/common/interface.js' -import setupFilesystem from '../../src/modules/filesystem/index.js' import setup from '@/common/index.js' describe('filesystem', () => { const testPath = mkTestPath('filesystem') - const create = async (): Promise<{ - filesystem: FilesystemContext - components: Components - socket: string - }> => { - const socket = Path.join(testPath, `${Math.random()}.socket`) + const create = async (): Promise<{ + filesystem: FilesystemContext + components: Components + socket: string + }> => { + const socket = Path.join(testPath, `${Math.random()}.socket`) const components = await setup({ socket }) - const filesystem = await setupFilesystem(components) + const filesystem = await setupFilesystem(components) - return { filesystem, components, socket } + return { filesystem, components, socket } } before(async () => { diff --git a/packages/daemon/test/modules/groups.spec.ts b/packages/daemon/test/modules/groups.spec.ts index 43bdaa6c..d1cdaeae 100644 --- a/packages/daemon/test/modules/groups.spec.ts +++ b/packages/daemon/test/modules/groups.spec.ts @@ -7,10 +7,10 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import setup from '@/common/index.js' import { createGroup } from '../utils/create-group.js' import { mkTestPath } from '../utils/paths.js' import type { Components } from '@/common/interface.js' +import setup from '@/common/index.js' describe('groups', () => { const testPath = mkTestPath('groups') @@ -23,12 +23,12 @@ describe('groups', () => { await fs.rm(testPath, { recursive: true }) }) - const create = async (): Promise<{ components: Components, socket: string }> => { - const socket = Path.join(testPath, `${Math.random()}.socket`) - const components = await setup({ socket }) + const create = async (): Promise<{ components: Components, socket: string }> => { + const socket = Path.join(testPath, `${Math.random()}.socket`) + const components = await setup({ socket }) - return { components, socket } - } + return { components, socket } + } it('creates a group', async () => { const { components } = await create() diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts index d5bd8ab3..6eadbba3 100644 --- a/packages/daemon/test/modules/network.spec.ts +++ b/packages/daemon/test/modules/network.spec.ts @@ -90,7 +90,7 @@ describe('network', () => { await Promise.all([ libp2p1.stop(), libp2p2.stop(), - components.stop() + components.stop() ]) }) @@ -138,7 +138,7 @@ describe('network', () => { assert.deepEqual(peer, libp2p.peerId.toBytes()) await libp2p.stop() - await components.stop() + await components.stop() }) // Something is failing inside websockets... @@ -168,11 +168,11 @@ describe('network', () => { await Promise.all([ libp2p1.stop(), libp2p2.stop(), - components.stop() + components.stop() ]) }) - it('libp2p remembers peers with persistant storage', async () => { + it.skip('libp2p remembers peers with persistant storage', async () => { const libp2p = await createLibp2p({}) const start = async (): Promise => setup({ socket, config: { private: false, storage: testPath } }) @@ -243,7 +243,7 @@ describe('network', () => { assert.deepEqual(connections[0].remotePeer.toBytes(), libp2p.peerId.toBytes()) await libp2p.stop() - await components.stop() + await components.stop() client.close() }) @@ -267,7 +267,7 @@ describe('network', () => { await helia.stop() await libp2p.stop() - await components.stop() + await components.stop() client.close() }) }) diff --git a/packages/daemon/test/modules/revisions.spec.ts b/packages/daemon/test/modules/revisions.spec.ts index fb23c95d..7c08f5e4 100644 --- a/packages/daemon/test/modules/revisions.spec.ts +++ b/packages/daemon/test/modules/revisions.spec.ts @@ -9,31 +9,31 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import setup from '@/common/index.js' -import setupRevisions from '@/modules/revisions/index.js' -import setupFilesystem from '@/modules/filesystem/index.js' -import type { Context as RevisionsContext } from '@/modules/revisions/index.js' -import type { Context as FilesystemContext } from '@/modules/filesystem/index.js' -import { mkTestPath } from '../utils/paths.js' -import type { Components } from '@/common/interface.js' import { createGroup } from '../utils/create-group.js' import { createDag } from '../utils/dag.js' +import { mkTestPath } from '../utils/paths.js' +import type { Components } from '@/common/interface.js' +import type { Context as FilesystemContext } from '@/modules/filesystem/index.js' +import type { Context as RevisionsContext } from '@/modules/revisions/index.js' +import setup from '@/common/index.js' +import setupFilesystem from '@/modules/filesystem/index.js' +import setupRevisions from '@/modules/revisions/index.js' describe('revisions', () => { const testPath = mkTestPath('revisions') const create = async (): Promise<{ - revisions: RevisionsContext - filesystem: FilesystemContext - components: Components - socket: string - }> => { - const socket = Path.join(testPath, `${Math.random()}.socket`) + revisions: RevisionsContext + filesystem: FilesystemContext + components: Components + socket: string + }> => { + const socket = Path.join(testPath, `${Math.random()}.socket`) const components = await setup({ socket }) - const filesystem = await setupFilesystem(components) - const revisions = await setupRevisions(components) + const filesystem = await setupFilesystem(components) + const revisions = await setupRevisions(components) - return { filesystem, revisions, components, socket } + return { filesystem, revisions, components, socket } } before(async () => { diff --git a/packages/daemon/test/modules/rpc.spec.ts b/packages/daemon/test/modules/rpc.spec.ts index 6623de46..2b707095 100644 --- a/packages/daemon/test/modules/rpc.spec.ts +++ b/packages/daemon/test/modules/rpc.spec.ts @@ -1,9 +1,9 @@ import assert from 'assert/strict' import fs from 'fs/promises' +import Path from 'path' import { createNetClient } from '@organicdesign/net-rpc' import { mkTestPath } from '../utils/paths.js' import setup from '@/common/index.js' -import Path from 'path' const testPath = mkTestPath('rpc') @@ -19,7 +19,7 @@ describe('rpc', () => { }) it('adds RPC methods', async () => { - const components = await setup({ socket }) + const components = await setup({ socket }) const testData = { key: 'value' } const returnData = { return: 'return-value' } @@ -40,6 +40,6 @@ describe('rpc', () => { assert.deepEqual(await methodPromise, testData) client.close() - await components.stop() + await components.stop() }) }) From 545701b63090240139bd0cd2c6ec69b7d5690ef7 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 28 Mar 2024 17:43:41 +1300 Subject: [PATCH 58/62] Fix libp2p peristent storage test. --- packages/daemon/test/modules/network.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts index 6eadbba3..f4dfa200 100644 --- a/packages/daemon/test/modules/network.spec.ts +++ b/packages/daemon/test/modules/network.spec.ts @@ -172,7 +172,7 @@ describe('network', () => { ]) }) - it.skip('libp2p remembers peers with persistant storage', async () => { + it('libp2p remembers peers with persistant storage', async () => { const libp2p = await createLibp2p({}) const start = async (): Promise => setup({ socket, config: { private: false, storage: testPath } }) @@ -190,7 +190,7 @@ describe('network', () => { const [saved] = await components.libp2p.peerStore.all() - assert.deepEqual(saved.id.toBytes(), peer.toBytes()) + assert.deepEqual(saved.id.toString(), peer.toString()) await components.stop() await libp2p.stop() From 1600ac9ab8edd55ba34f2ff8ee71271d0abcaea8 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 29 Mar 2024 08:58:22 +1300 Subject: [PATCH 59/62] Fix datastore/blockstore cleanup. --- packages/daemon/src/common/index.ts | 33 +++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 7019ab59..8b2f2cbc 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -142,13 +142,32 @@ export default async (settings: Partial = {}): Promise => { const stop = async (): Promise => { logger.info('cleaning up...') - await net.close() - await tick.stop() - await downloader.stop() - await groups.stop() - await welo.stop() - await helia.stop() - await libp2p.stop() + const closeDatastore = async (): Promise => { + if (datastore instanceof FsDatastore) { + await datastore.close() + } + } + + const closeBlockstore = async (): Promise => { + if (blockstore instanceof FsBlockstore) { + await blockstore.close() + } + } + + await Promise.all([ + net.close(), + tick.stop(), + downloader.stop(), + groups.stop(), + welo.stop(), + helia.stop(), + libp2p.stop() + ]) + + await Promise.all([ + closeDatastore, + closeBlockstore + ]) logger.info('exiting...') } From 427e2edc7272b6c13e0491f06229c124ed594146 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 29 Mar 2024 09:15:12 +1300 Subject: [PATCH 60/62] Add keyManager option to components setup. --- packages/daemon/src/common/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts index 8b2f2cbc..36f62be2 100644 --- a/packages/daemon/src/common/index.ts +++ b/packages/daemon/src/common/index.ts @@ -1,7 +1,7 @@ import Path from 'path' import { bitswap } from '@helia/block-brokers' import HeliaPinManager from '@organicdesign/db-helia-pin-manager' -import { createKeyManager } from '@organicdesign/db-key-manager' +import { createKeyManager, type KeyManager } from '@organicdesign/db-key-manager' import { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' import { createNetServer } from '@organicdesign/net-rpc' import { MemoryBlockstore } from 'blockstore-core' @@ -28,18 +28,18 @@ interface Setup { socket: string config: Record key?: string + keyManager: KeyManager } export default async (settings: Partial = {}): Promise => { - const setup: Setup = { + const setup: Pick = { socket: settings.socket ?? '/tmp/server.socket', - config: settings.config ?? {}, - key: settings.key + config: settings.config ?? {} } const logger = createLogger('common') const parseConfig = (shape: T): z.infer => shape.parse(setup.config) - const keyManager = await createKeyManager(setup.key) + const keyManager = settings.keyManager ?? await createKeyManager(settings.key) const net = await createNetServer(setup.socket) const config = parseConfig(Config) const events = new EventTarget() From 738f05bb103a85108c7fd4e03f5675967d6de919 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 29 Mar 2024 09:17:16 +1300 Subject: [PATCH 61/62] Fix libp2p persistent storage test. --- packages/daemon/test/modules/network.spec.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts index f4dfa200..330b6c57 100644 --- a/packages/daemon/test/modules/network.spec.ts +++ b/packages/daemon/test/modules/network.spec.ts @@ -2,6 +2,7 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' import { unixfs } from '@helia/unixfs' +import { createKeyManager } from '@organicdesign/db-key-manager' import { createNetClient } from '@organicdesign/net-rpc' import { createHelia } from 'helia' import { Key } from 'interface-datastore' @@ -174,7 +175,16 @@ describe('network', () => { it('libp2p remembers peers with persistant storage', async () => { const libp2p = await createLibp2p({}) - const start = async (): Promise => setup({ socket, config: { private: false, storage: testPath } }) + const keyManager = await createKeyManager() + + const start = async (): Promise => setup({ + socket, + keyManager, + config: { + private: false, + storage: testPath + } + }) let components = await start() @@ -190,7 +200,7 @@ describe('network', () => { const [saved] = await components.libp2p.peerStore.all() - assert.deepEqual(saved.id.toString(), peer.toString()) + assert.deepEqual(saved.id.toBytes(), peer.toBytes()) await components.stop() await libp2p.stop() From 5b397e119e6c92ce13062e0c0a7936de8f1cd240 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 29 Mar 2024 09:43:04 +1300 Subject: [PATCH 62/62] Add test output to git ignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ccb1df37..f277a47b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /data /config +/packages/*/test-out build dist