diff --git a/.gitignore b/.gitignore index ccb1df37..f277a47b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /data /config +/packages/*/test-out build dist diff --git a/packages/daemon/src/common/commands/addresses.ts b/packages/daemon/src/common/commands/addresses.ts new file mode 100644 index 00000000..aeb3c5fe --- /dev/null +++ b/packages/daemon/src/common/commands/addresses.ts @@ -0,0 +1,10 @@ +import { Addresses } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, libp2p }) => { + net.rpc.addMethod(Addresses.name, async (): Promise => { + return libp2p.getMultiaddrs().map(a => a.toString()) + }) +} + +export default command diff --git a/packages/daemon/src/modules/network/commands/connect.ts b/packages/daemon/src/common/commands/connect.ts similarity index 54% rename from packages/daemon/src/modules/network/commands/connect.ts rename to packages/daemon/src/common/commands/connect.ts index f404843f..7642083f 100644 --- a/packages/daemon/src/modules/network/commands/connect.ts +++ b/packages/daemon/src/common/commands/connect.ts @@ -1,14 +1,13 @@ import { multiaddr } from '@multiformats/multiaddr' import { Connect } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Connect.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, libp2p }) => { + net.rpc.addMethod(Connect.name, async (raw: unknown): Promise => { const params = Connect.Params.parse(raw) const address = multiaddr(params.address) - await context.libp2p.dial(address) + await libp2p.dial(address) return null }) diff --git a/packages/daemon/src/common/commands/connections.ts b/packages/daemon/src/common/commands/connections.ts new file mode 100644 index 00000000..381bade5 --- /dev/null +++ b/packages/daemon/src/common/commands/connections.ts @@ -0,0 +1,10 @@ +import { Connections } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, libp2p }) => { + net.rpc.addMethod(Connections.name, async (): Promise => { + return libp2p.getConnections().map(c => c.remoteAddr.toString()) + }) +} + +export default command diff --git a/packages/daemon/src/modules/network/commands/count-peers.ts b/packages/daemon/src/common/commands/count-peers.ts similarity index 74% rename from packages/daemon/src/modules/network/commands/count-peers.ts rename to packages/daemon/src/common/commands/count-peers.ts index 3966b7d5..f59f9fd0 100644 --- a/packages/daemon/src/modules/network/commands/count-peers.ts +++ b/packages/daemon/src/common/commands/count-peers.ts @@ -1,14 +1,13 @@ import { CountPeers } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(CountPeers.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, libp2p }) => { + net.rpc.addMethod(CountPeers.name, async (raw: unknown): Promise => { const countPeers = async (cid: CID, options?: { timeout: number }): Promise => { let count = 0 - const itr = context.libp2p.contentRouting.findProviders(cid, { + const itr = libp2p.contentRouting.findProviders(cid, { signal: AbortSignal.timeout(options?.timeout ?? 3000) }) diff --git a/packages/daemon/src/modules/groups/commands/create-group.ts b/packages/daemon/src/common/commands/create-group.ts similarity index 57% rename from packages/daemon/src/modules/groups/commands/create-group.ts rename to packages/daemon/src/common/commands/create-group.ts index f8de7451..d5b8be46 100644 --- a/packages/daemon/src/modules/groups/commands/create-group.ts +++ b/packages/daemon/src/common/commands/create-group.ts @@ -1,23 +1,22 @@ import { CreateGroup } from '@organicdesign/db-rpc-interfaces' import { fromString as uint8ArrayFromString } from 'uint8arrays' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(CreateGroup.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, welo, groups }) => { + net.rpc.addMethod(CreateGroup.name, async (raw: unknown): Promise => { const params = CreateGroup.Params.parse(raw) const peerValues = params.peers.map(p => uint8ArrayFromString(p, 'base58btc')) - const manifest = await context.welo.determine({ + const manifest = await welo.determine({ name: params.name, meta: { type: 'group' }, access: { protocol: '/hldb/access/static', - config: { write: [context.welo.identity.id, ...peerValues] } + config: { write: [welo.identity.id, ...peerValues] } } }) - await context.groups.add(manifest) + await groups.add(manifest) return manifest.address.cid.toString() }) diff --git a/packages/daemon/src/modules/downloader/commands/get-speeds.ts b/packages/daemon/src/common/commands/get-speeds.ts similarity index 57% rename from packages/daemon/src/modules/downloader/commands/get-speeds.ts rename to packages/daemon/src/common/commands/get-speeds.ts index a612f37c..8cd35ae1 100644 --- a/packages/daemon/src/modules/downloader/commands/get-speeds.ts +++ b/packages/daemon/src/common/commands/get-speeds.ts @@ -1,15 +1,14 @@ import { GetSpeeds } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(GetSpeeds.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, pinManager }) => { + net.rpc.addMethod(GetSpeeds.name, async (raw: unknown): Promise => { const params = GetSpeeds.Params.parse(raw) return Promise.all(params.cids.map(async str => { const cid = CID.parse(str) - const speed = await context.pinManager.getSpeed(cid, params.range) + const speed = await pinManager.getSpeed(cid, params.range) return { cid: str, diff --git a/packages/daemon/src/modules/downloader/commands/get-status.ts b/packages/daemon/src/common/commands/get-status.ts similarity index 58% rename from packages/daemon/src/modules/downloader/commands/get-status.ts rename to packages/daemon/src/common/commands/get-status.ts index a127b3b8..ffbc5178 100644 --- a/packages/daemon/src/modules/downloader/commands/get-status.ts +++ b/packages/daemon/src/common/commands/get-status.ts @@ -1,19 +1,18 @@ import { GetStatus } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(GetStatus.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ pinManager, net }) => { + net.rpc.addMethod(GetStatus.name, async (raw: unknown): Promise => { const params = GetStatus.Params.parse(raw) return Promise.all(params.cids.map(async str => { const cid = CID.parse(str) const [state, blocks, size] = await Promise.all([ - context.pinManager.getState(cid), - context.pinManager.getBlockCount(cid), - context.pinManager.getSize(cid) + pinManager.getState(cid), + pinManager.getBlockCount(cid), + pinManager.getSize(cid) ]) return { diff --git a/packages/daemon/src/common/commands/id.ts b/packages/daemon/src/common/commands/id.ts new file mode 100644 index 00000000..5e497ece --- /dev/null +++ b/packages/daemon/src/common/commands/id.ts @@ -0,0 +1,11 @@ +import { ID } from '@organicdesign/db-rpc-interfaces' +import { toString as uint8ArrayToString } from 'uint8arrays' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ net, welo }) => { + net.rpc.addMethod(ID.name, async (): Promise => { + return uint8ArrayToString(welo.identity.id, 'base58btc') + }) +} + +export default command diff --git a/packages/daemon/src/modules/groups/commands/join-group.ts b/packages/daemon/src/common/commands/join-group.ts similarity index 55% rename from packages/daemon/src/modules/groups/commands/join-group.ts rename to packages/daemon/src/common/commands/join-group.ts index 7b8553a1..e0233761 100644 --- a/packages/daemon/src/modules/groups/commands/join-group.ts +++ b/packages/daemon/src/common/commands/join-group.ts @@ -1,15 +1,14 @@ import { JoinGroup } from '@organicdesign/db-rpc-interfaces' import { Address } from 'welo' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(JoinGroup.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, welo, groups }) => { + net.rpc.addMethod(JoinGroup.name, async (raw: unknown): Promise => { const params = JoinGroup.Params.parse(raw) - const manifest = await context.welo.fetch(Address.fromString(`/hldb/${params.group}`)) + const manifest = await welo.fetch(Address.fromString(`/hldb/${params.group}`)) try { - await context.groups.add(manifest) + await groups.add(manifest) } catch (error) { if ((error as Error).message.includes('is already open')) { throw new Error('group has already been joined') diff --git a/packages/daemon/src/modules/groups/commands/list-groups.ts b/packages/daemon/src/common/commands/list-groups.ts similarity index 53% rename from packages/daemon/src/modules/groups/commands/list-groups.ts rename to packages/daemon/src/common/commands/list-groups.ts index b6bdde3c..e28ed8b6 100644 --- a/packages/daemon/src/modules/groups/commands/list-groups.ts +++ b/packages/daemon/src/common/commands/list-groups.ts @@ -1,12 +1,11 @@ import { ListGroups } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(ListGroups.name, async (): Promise => { +const command: ModuleMethod = ({ net, groups }) => { + net.rpc.addMethod(ListGroups.name, async (): Promise => { const promises: Array<{ group: string, name: string }> = [] - for (const { key: cid, value: database } of context.groups.all()) { + for (const { key: cid, value: database } of groups.all()) { promises.push({ group: cid, name: database.manifest.name }) } diff --git a/packages/daemon/src/modules/downloader/commands/set-priority.ts b/packages/daemon/src/common/commands/set-priority.ts similarity index 56% rename from packages/daemon/src/modules/downloader/commands/set-priority.ts rename to packages/daemon/src/common/commands/set-priority.ts index 9a845fef..9331a17c 100644 --- a/packages/daemon/src/modules/downloader/commands/set-priority.ts +++ b/packages/daemon/src/common/commands/set-priority.ts @@ -1,13 +1,12 @@ import Path from 'path' import { SetPriority } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(SetPriority.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, pinManager }) => { + net.rpc.addMethod(SetPriority.name, async (raw: unknown): Promise => { const params = SetPriority.Params.parse(raw) const key = Path.join('/', params.group, params.path) - const pinInfo = await context.pinManager.get(key) + const pinInfo = await pinManager.get(key) if (pinInfo == null) { throw new Error('no such pin') @@ -15,7 +14,7 @@ const command: ModuleMethod = (context, { rpc }) => { pinInfo.priority = params.priority - await context.pinManager.put(key, pinInfo) + await pinManager.put(key, pinInfo) return null }) diff --git a/packages/daemon/src/common/commands/sneakernet-receive.ts b/packages/daemon/src/common/commands/sneakernet-receive.ts new file mode 100644 index 00000000..2d909b34 --- /dev/null +++ b/packages/daemon/src/common/commands/sneakernet-receive.ts @@ -0,0 +1,14 @@ +import { SneakernetReveive } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ sneakernet, net }) => { + net.rpc.addMethod(SneakernetReveive.name, async (raw: unknown): Promise => { + const params = SneakernetReveive.Params.parse(raw) + + await sneakernet.import(params.path) + + return null + }) +} + +export default command diff --git a/packages/daemon/src/common/commands/sneakernet-send.ts b/packages/daemon/src/common/commands/sneakernet-send.ts new file mode 100644 index 00000000..b6b18c31 --- /dev/null +++ b/packages/daemon/src/common/commands/sneakernet-send.ts @@ -0,0 +1,14 @@ +import { SneakernetSend } from '@organicdesign/db-rpc-interfaces' +import type { ModuleMethod } from '@/interface.js' + +const command: ModuleMethod = ({ sneakernet, net }) => { + net.rpc.addMethod(SneakernetSend.name, async (raw: unknown): Promise => { + const params = SneakernetSend.Params.parse(raw) + + await sneakernet.export(params.path, params.peers) + + return null + }) +} + +export default command diff --git a/packages/daemon/src/modules/groups/commands/sync.ts b/packages/daemon/src/common/commands/sync.ts similarity index 83% rename from packages/daemon/src/modules/groups/commands/sync.ts rename to packages/daemon/src/common/commands/sync.ts index 6bc4a00c..e6d6392e 100644 --- a/packages/daemon/src/modules/groups/commands/sync.ts +++ b/packages/daemon/src/common/commands/sync.ts @@ -2,7 +2,6 @@ import { Sync } from '@organicdesign/db-rpc-interfaces' import { HeadsExchange } from 'welo/utils/heads-exchange' import { cidstring } from 'welo/utils/index' import { getHeads, addHeads } from 'welo/utils/replicator' -import type { Provides, Requires } from '../index.js' import type { ModuleMethod } from '@/interface.js' import type { Peer, Libp2p } from '@libp2p/interface' import type { Database } from 'welo' @@ -59,18 +58,18 @@ const sync = async (libp2p: Libp2p, peer: Peer, database: Database, options: Par await stream.close() } -const command: ModuleMethod = (context, { rpc, network }) => { - rpc.addMethod(Sync.name, async (): Promise => { - const peers = network.libp2p.getPeers() - const databases = context.welo.opened.values() +const command: ModuleMethod = ({ libp2p, welo, net }) => { + net.rpc.addMethod(Sync.name, async (): Promise => { + const peers = libp2p.getPeers() + const databases = welo.opened.values() const promises: Array> = [] for (const peerId of peers) { - const peer = await network.libp2p.peerStore.get(peerId) + const peer = await libp2p.peerStore.get(peerId) for (const database of databases) { - promises.push(sync(network.libp2p, peer, database)) + promises.push(sync(libp2p, peer, database)) } } diff --git a/packages/daemon/src/common/downloader/index.ts b/packages/daemon/src/common/downloader/index.ts new file mode 100644 index 00000000..af22abf3 --- /dev/null +++ b/packages/daemon/src/common/downloader/index.ts @@ -0,0 +1,122 @@ +import parallel from 'it-parallel' +import { pipe } from 'it-pipe' +import { collect } from 'streaming-iterables' +import { Event, EventTarget } from 'ts-event-target' +import { linearWeightTranslation } from './utils.js' +import type { PinManager } from '../pin-manager/index.js' +import type { Startable } from '@libp2p/interface' +import type { CID } from 'multiformats/cid' + +export class DownloadErrorEvent extends Event<'download:error'> { + readonly error: Error + + constructor (error: Error) { + super('download:error') + + this.error = error + } +} + +export class Downloader implements Startable { + private readonly slots: number + private readonly pinManager: PinManager + private controller: AbortController = new AbortController() + private loopPromise: Promise | null = null + readonly events = new EventTarget<[DownloadErrorEvent]>() + + constructor (pinManager: PinManager, slots: number) { + this.slots = slots + this.pinManager = pinManager + } + + async start (): Promise { + await this.loopPromise + this.controller = new AbortController() + this.loopPromise = this.loop() + } + + async stop (): Promise { + this.controller.abort() + await this.loopPromise + } + + private async loop (): Promise { + for (;;) { + if (this.isAborted) { + return + } + + await pipe( + () => this.delay(), + i => this.getPins(i), + i => this.batchDownload(i), + i => parallel(i, { concurrency: this.slots, ordered: false }), + i => this.catcher(i), + async i => collect(i) + ) + + await new Promise(resolve => setTimeout(resolve, 100)) + } + } + + private async * batchDownload (itr: AsyncIterable<[CID, number]>): AsyncGenerator<() => Promise<{ cid: CID, block: Uint8Array }>, void, undefined> { + for await (const [cid, priority] of itr) { + if (this.isAborted) { + return + } + + const weight = Math.floor(linearWeightTranslation(priority / 100) * this.slots) + 1 + const downloaders = await this.pinManager.download(cid, { limit: weight }) + + yield * downloaders + } + } + + private async * catcher (itr: AsyncIterable): AsyncIterable { + try { + yield * itr + } catch (e) { + const error = e instanceof Error ? e : new Error(JSON.stringify(e)) + + this.events.dispatchEvent(new DownloadErrorEvent(error)) + + yield * this.catcher(itr) + } + } + + private async * delay (): AsyncGenerator { + for (;;) { + if (this.isAborted) { + return + } + + await new Promise(resolve => setTimeout(resolve, 100)) + yield + } + } + + async * getPins (loop: AsyncIterable): AsyncGenerator<[CID, number]> { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _ of loop) { + for await (const { value } of this.pinManager.getActive()) { + if (this.isAborted) { + return + } + + yield [value.cid, value.priority] + } + } + } + + private get isAborted (): boolean { + return this.controller.signal.aborted + } +} + +export const createDownloader = async (pinManage: PinManager, slots: number): Promise => { + const downloader = new Downloader(pinManage, slots) + + await downloader.start() + + return downloader +} diff --git a/packages/daemon/src/common/downloader/utils.ts b/packages/daemon/src/common/downloader/utils.ts new file mode 100644 index 00000000..2b77bf57 --- /dev/null +++ b/packages/daemon/src/common/downloader/utils.ts @@ -0,0 +1,7 @@ +export const linearWeightTranslation = (p: number): number => { + return 1 - p +} + +export const logWeightTranslation = (p: number): number => { + return 1 - Math.log10((10 - 1) * p - 1) +} diff --git a/packages/daemon/src/modules/groups/entry-tracker.ts b/packages/daemon/src/common/entry-tracker.ts similarity index 100% rename from packages/daemon/src/modules/groups/entry-tracker.ts rename to packages/daemon/src/common/entry-tracker.ts diff --git a/packages/daemon/src/modules/groups/groups.ts b/packages/daemon/src/common/groups.ts similarity index 84% rename from packages/daemon/src/modules/groups/groups.ts rename to packages/daemon/src/common/groups.ts index 35e099c6..0b95e682 100644 --- a/packages/daemon/src/modules/groups/groups.ts +++ b/packages/daemon/src/common/groups.ts @@ -1,7 +1,7 @@ import { Key } from 'interface-datastore' +import { Event, EventTarget } from 'ts-event-target' import { Manifest } from 'welo/manifest/index' import { decodeCbor } from 'welo/utils/block' -import { logger } from './index.js' import type { Pair, KeyvalueDB } from '@/interface.js' import type { Startable } from '@libp2p/interfaces/startable' import type { Datastore } from 'interface-datastore' @@ -14,11 +14,24 @@ export interface Components { welo: Welo } +type EventTypes = 'groups:joined' + +export class GroupEvent extends Event { + readonly cid: CID + + constructor (type: EventTypes, cid: CID) { + super(type) + + this.cid = cid + } +} + export class Groups implements Startable { private readonly welo: Welo private readonly datastore: Datastore private readonly groups = new Map() private started = false + readonly events = new EventTarget<[GroupEvent]>() constructor (components: Components) { this.welo = components.welo @@ -61,7 +74,7 @@ export class Groups implements Startable { await this.datastore.put(new Key(database.address.cid.toString()), database.manifest.block.bytes) - logger.info(`[groups] [join] ${manifest.address.cid.toString()}`) + this.events.dispatchEvent(new GroupEvent('groups:joined', manifest.address.cid)) } get (group: CID): KeyvalueDB | undefined { diff --git a/packages/daemon/src/common/handle-commands.ts b/packages/daemon/src/common/handle-commands.ts new file mode 100644 index 00000000..e6a9f20d --- /dev/null +++ b/packages/daemon/src/common/handle-commands.ts @@ -0,0 +1,38 @@ +import addresses from './commands/addresses.js' +import connect from './commands/connect.js' +import connections from './commands/connections.js' +import countPeers from './commands/count-peers.js' +import createGroup from './commands/create-group.js' +import getSpeeds from './commands/get-speeds.js' +import getStatus from './commands/get-status.js' +import id from './commands/id.js' +import joinGroup from './commands/join-group.js' +import listGroups from './commands/list-groups.js' +import setPriority from './commands/set-priority.js' +import sneakernetReveive from './commands/sneakernet-receive.js' +import sneakernetSend from './commands/sneakernet-send.js' +import sync from './commands/sync.js' +import type { Components } from './interface.js' + +export default (components: Components): void => { + const commands = [ + addresses, + connect, + connections, + countPeers, + createGroup, + getSpeeds, + getStatus, + id, + joinGroup, + listGroups, + setPriority, + sneakernetReveive, + sneakernetSend, + sync + ] + + for (const command of commands) { + command(components, {}) + } +} diff --git a/packages/daemon/src/common/index.ts b/packages/daemon/src/common/index.ts new file mode 100644 index 00000000..36f62be2 --- /dev/null +++ b/packages/daemon/src/common/index.ts @@ -0,0 +1,198 @@ +import Path from 'path' +import { bitswap } from '@helia/block-brokers' +import HeliaPinManager from '@organicdesign/db-helia-pin-manager' +import { createKeyManager, type KeyManager } from '@organicdesign/db-key-manager' +import { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' +import { createNetServer } from '@organicdesign/net-rpc' +import { MemoryBlockstore } from 'blockstore-core' +import { FsBlockstore } from 'blockstore-fs' +import { MemoryDatastore } from 'datastore-core' +import { FsDatastore } from 'datastore-fs' +import { createHelia } from 'helia' +import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' +import { type z } from 'zod' +import { createDownloader } from './downloader/index.js' +import { EntryTracker } from './entry-tracker.js' +import { createGroups } from './groups.js' +import handleCommands from './handle-commands.js' +import { Config, type Components } from './interface.js' +import createLibp2p from './libp2p.js' +import { PinManager } from './pin-manager/index.js' +import { Sneakernet } from './sneakernet/index.js' +import { createTick } from './tick.js' +import type { KeyvalueDB } from '@/interface.js' +import { createLogger } from '@/logger.js' +import { isMemory, extendDatastore } from '@/utils.js' + +interface Setup { + socket: string + config: Record + key?: string + keyManager: KeyManager +} + +export default async (settings: Partial = {}): Promise => { + const setup: Pick = { + socket: settings.socket ?? '/tmp/server.socket', + config: settings.config ?? {} + } + + const logger = createLogger('common') + const parseConfig = (shape: T): z.infer => shape.parse(setup.config) + const keyManager = settings.keyManager ?? await createKeyManager(settings.key) + const net = await createNetServer(setup.socket) + const config = parseConfig(Config) + const events = new EventTarget() + + const datastore = isMemory(config.storage) + ? new MemoryDatastore() + : new FsDatastore(Path.join(config.storage, 'datastore')) + + const blockstore = isMemory(config.storage) + ? new MemoryBlockstore() + : new FsBlockstore(Path.join(config.storage, 'blockstore')) + + const peerId = await keyManager.getPeerId() + const psk = keyManager.getPskKey() + + const libp2p = await createLibp2p({ + datastore: extendDatastore(datastore, 'datastore/libp2p'), + psk: config.private ? psk : undefined, + peerId, + ...config + }) + + const manualBlockBroker = new ManualBlockBroker() + + const helia = await createHelia({ + datastore: extendDatastore(datastore, 'helia/datastore'), + libp2p, + blockstore, + blockBrokers: [bitswap(), () => manualBlockBroker] + }) + + const welo = await createWelo({ + // @ts-expect-error Helia version mismatch here. + ipfs: helia, + replicators: [bootstrapReplicator(), pubsubReplicator()], + identity: await keyManager.getWeloIdentity() + }) + + const groups = await createGroups({ + datastore: extendDatastore(datastore, 'groups'), + welo + }) + + groups.events.addEventListener('groups:joined', ({ cid }) => { + logger.info(`[groups] [join] ${cid.toString()}`) + }) + + const sneakernet = new Sneakernet({ + welo, + helia, + datastore: extendDatastore(datastore, 'sneakernet'), + manualBlockBroker, + groups + }) + + const getTracker = (database: KeyvalueDB): EntryTracker => new EntryTracker(database) + + const heliaPinManager = new HeliaPinManager({ + helia, + datastore: extendDatastore(datastore, 'heliaPinManager') + }) + + heliaPinManager.events.addEventListener('downloads:added', ({ cid }) => { + logger.info(`[downloads] [+] ${cid}`) + }) + + heliaPinManager.events.addEventListener('pins:added', ({ cid }) => { + logger.info(`[pins] [+] ${cid}`) + }) + + heliaPinManager.events.addEventListener('pins:adding', ({ cid }) => { + logger.info(`[pins] [~] ${cid}`) + }) + + heliaPinManager.events.addEventListener('pins:removed', ({ cid }) => { + logger.info(`[pins] [-] ${cid}`) + }) + + const tick = await createTick(config.tickInterval) + + tick.events.addEventListener('method:error', ({ error }) => { + logger.error('tick: ', error) + }) + + const pinManager = new PinManager({ + pinManager: heliaPinManager, + datastore: extendDatastore(datastore, 'pinManager') + }) + + pinManager.events.addEventListener('reference:removed', ({ key }) => { + logger.info(`[references] [-] ${key}`) + }) + + const downloader = await createDownloader(pinManager, config.slots) + + downloader.events.addEventListener('download:error', ({ error }) => { + logger.error('downloader: ', error) + }) + + const stop = async (): Promise => { + logger.info('cleaning up...') + + const closeDatastore = async (): Promise => { + if (datastore instanceof FsDatastore) { + await datastore.close() + } + } + + const closeBlockstore = async (): Promise => { + if (blockstore instanceof FsBlockstore) { + await blockstore.close() + } + } + + await Promise.all([ + net.close(), + tick.stop(), + downloader.stop(), + groups.stop(), + welo.stop(), + helia.stop(), + libp2p.stop() + ]) + + await Promise.all([ + closeDatastore, + closeBlockstore + ]) + + logger.info('exiting...') + } + + const components: Components = { + sneakernet, + getTracker, + helia, + libp2p, + blockstore, + datastore, + net, + tick, + downloader, + parseConfig, + stop, + groups, + pinManager, + welo, + heliaPinManager, + events, + keyManager + } + + handleCommands(components) + + return components +} diff --git a/packages/daemon/src/common/interface.ts b/packages/daemon/src/common/interface.ts new file mode 100644 index 00000000..f14bc9fb --- /dev/null +++ b/packages/daemon/src/common/interface.ts @@ -0,0 +1,52 @@ +import { z } from 'zod' +import type { Downloader } from './downloader/index.js' +import type { EntryTracker } from './entry-tracker.js' +import type { Groups } from './groups.js' +import type { PinManager } from './pin-manager/index.js' +import type { Sneakernet } from './sneakernet/index.js' +import type { Tick } from './tick.js' +import type { KeyvalueDB } from '@/interface.js' +import type { Helia } from '@helia/interface' +import type { Libp2p } from '@libp2p/interface' +import type HeliaPinManager from '@organicdesign/db-helia-pin-manager' +import type { KeyManager } from '@organicdesign/db-key-manager' +import type { NetServer } from '@organicdesign/net-rpc' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' +import type { Welo } from 'welo' + +export const Config = z.object({ + storage: z.string().default(':memory:'), + slots: z.number().int().min(1).max(100).default(20), + tickInterval: z.number().default(10 * 60), + serverMode: z.boolean().default(false), + private: z.boolean().default(false), + bootstrap: z.array(z.string()).default([]), + addresses: z.array(z.string()).default([ + '/ip4/127.0.0.1/tcp/0', + '/ip4/127.0.0.1/tcp/0/ws' + ]) +}) + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export type Config = z.output + +export interface Components { + helia: Helia + libp2p: Libp2p + welo: Welo + datastore: Datastore + blockstore: Blockstore + stop(): Promise + net: NetServer + tick: Tick + sneakernet: Sneakernet + getTracker(keyvalueDB: KeyvalueDB): EntryTracker + parseConfig(shape: T): z.infer + downloader: Downloader + groups: Groups + pinManager: PinManager + heliaPinManager: HeliaPinManager + events: EventTarget + keyManager: KeyManager +} diff --git a/packages/daemon/src/modules/network/libp2p.ts b/packages/daemon/src/common/libp2p.ts similarity index 100% rename from packages/daemon/src/modules/network/libp2p.ts rename to packages/daemon/src/common/libp2p.ts diff --git a/packages/daemon/src/modules/downloader/pin-manager.ts b/packages/daemon/src/common/pin-manager/index.ts similarity index 84% rename from packages/daemon/src/modules/downloader/pin-manager.ts rename to packages/daemon/src/common/pin-manager/index.ts index e441570d..42f470eb 100644 --- a/packages/daemon/src/modules/downloader/pin-manager.ts +++ b/packages/daemon/src/common/pin-manager/index.ts @@ -1,16 +1,33 @@ import { Key, type Datastore } from 'interface-datastore' import all from 'it-all' -import { type CID } from 'multiformats/cid' +import { Event, EventTarget } from 'ts-event-target' import { encodePinInfo, decodePinInfo } from './utils.js' -import { logger } from './index.js' import type { PinInfo } from './interface.js' import type { Pair } from '@/interface.js' import type { BlockInfo } from '@organicdesign/db-helia-pin-manager' import type HeliaPinManager from '@organicdesign/db-helia-pin-manager' +import type { CID } from 'multiformats/cid' + +type EventTypes = 'reference:removed' + +export class PinManagerEvent extends Event { + readonly key: string + readonly priority: number + readonly cid: CID + + constructor (type: EventTypes, data: { key: string } & PinInfo) { + super(type) + + this.key = data.key + this.cid = data.cid + this.priority = data.priority + } +} export class PinManager { private readonly datastore: Datastore private readonly pinManager: HeliaPinManager + readonly events = new EventTarget<[PinManagerEvent]>() constructor (components: { datastore: Datastore, pinManager: HeliaPinManager }) { this.datastore = components.datastore @@ -82,7 +99,7 @@ export class PinManager { await this.pinManager.unpin(pinInfo.cid) } - logger.info(`[references] [-] ${key.toString()}`) + this.events.dispatchEvent(new PinManagerEvent('reference:removed', { key, ...pinInfo })) await this.datastore.delete(new Key(key)) } diff --git a/packages/daemon/src/modules/downloader/interface.ts b/packages/daemon/src/common/pin-manager/interface.ts similarity index 100% rename from packages/daemon/src/modules/downloader/interface.ts rename to packages/daemon/src/common/pin-manager/interface.ts diff --git a/packages/daemon/src/modules/downloader/utils.ts b/packages/daemon/src/common/pin-manager/utils.ts similarity index 79% rename from packages/daemon/src/modules/downloader/utils.ts rename to packages/daemon/src/common/pin-manager/utils.ts index e845b3eb..71f75b47 100644 --- a/packages/daemon/src/modules/downloader/utils.ts +++ b/packages/daemon/src/common/pin-manager/utils.ts @@ -2,14 +2,6 @@ import { CID } from 'multiformats/cid' import { EncodedPinInfo, type PinInfo } from './interface.js' import { decodeAny, encodeAny } from '@/utils.js' -export const linearWeightTranslation = (p: number): number => { - return 1 - p -} - -export const logWeightTranslation = (p: number): number => { - return 1 - Math.log10((10 - 1) * p - 1) -} - export const decodePinInfo = (data: Uint8Array): PinInfo | null => { const obj = decodeAny(data) diff --git a/packages/daemon/src/modules/sneakernet/sneakernet.ts b/packages/daemon/src/common/sneakernet/index.ts similarity index 86% rename from packages/daemon/src/modules/sneakernet/sneakernet.ts rename to packages/daemon/src/common/sneakernet/index.ts index 2b453e8c..076680bb 100644 --- a/packages/daemon/src/modules/sneakernet/sneakernet.ts +++ b/packages/daemon/src/common/sneakernet/index.ts @@ -11,10 +11,20 @@ import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { getHeads } from 'welo/utils/replicator' import { EncodedPeerData, PeerData } from './interface.js' -import type { Requires } from './index.js' +import type { Groups } from '../groups.js' import type { KeyvalueDB, Pair } from '@/interface.js' +import type { Helia } from '@helia/interface' import type { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' import type { Blockstore } from 'interface-blockstore' +import type { Welo } from 'welo' + +export interface Components { + helia: Helia + welo: Welo + datastore: Datastore + manualBlockBroker: ManualBlockBroker + groups: Groups +} export class Sneakernet { private readonly datastore: Datastore @@ -24,13 +34,13 @@ export class Sneakernet { private readonly getGroups: () => Array> private readonly car: Car - constructor (components: Requires, datastore: Datastore) { - this.datastore = datastore - this.id = components.groups.welo.identity.id - this.broker = components.network.manualBlockBroker - this.getGroups = () => [...components.groups.groups.all()] - this.car = car(components.network.helia) - this.blockstore = components.base.blockstore + constructor (components: Components) { + this.datastore = components.datastore + this.id = components.welo.identity.id + this.broker = components.manualBlockBroker + this.getGroups = () => [...components.groups.all()] + this.car = car(components.helia) + this.blockstore = components.helia.blockstore } async export (path: string, peers: string[] = []): Promise { diff --git a/packages/daemon/src/modules/sneakernet/interface.ts b/packages/daemon/src/common/sneakernet/interface.ts similarity index 100% rename from packages/daemon/src/modules/sneakernet/interface.ts rename to packages/daemon/src/common/sneakernet/interface.ts diff --git a/packages/daemon/src/common/tick.ts b/packages/daemon/src/common/tick.ts new file mode 100644 index 00000000..3e00ae01 --- /dev/null +++ b/packages/daemon/src/common/tick.ts @@ -0,0 +1,97 @@ +import { Event, EventTarget } from 'ts-event-target' +import type { Startable } from '@libp2p/interface' + +export class MethodErrorEvent extends Event<'method:error'> { + readonly error: Error + + constructor (error: Error) { + super('method:error') + + this.error = error + } +} + +interface Method { (signal?: AbortSignal): any } + +export class Tick implements Startable { + private readonly interval: number + private readonly methods: Method[] = [] + private controller: AbortController = new AbortController() + private loopPromise: Promise | null = null + readonly events = new EventTarget<[MethodErrorEvent]>() + + constructor (interval: number) { + this.interval = interval + } + + add (method: Method): void { + this.methods.push(method) + } + + async start (): Promise { + await this.loopPromise + this.controller = new AbortController() + this.loopPromise = this.loop() + } + + async stop (): Promise { + this.controller.abort() + + await this.loopPromise + } + + private get signal (): AbortSignal { + return this.controller.signal + } + + private get isAborted (): boolean { + return this.controller.signal.aborted + } + + private async loop (): Promise { + for (;;) { + for (const method of this.methods) { + try { + await method(this.signal) + } catch (e) { + const error = e instanceof Error ? e : new Error(JSON.stringify(e)) + + this.events.dispatchEvent(new MethodErrorEvent(error)) + } + + if (this.isAborted) { + return + } + } + + await new Promise(resolve => { + const listener = (): void => { + if (timeout != null) { + clearTimeout(timeout) + this.signal.removeEventListener('abort', listener) + resolve() + } + } + + const timeout = setTimeout(() => { + this.signal.removeEventListener('abort', listener) + resolve() + }, this.interval) + + this.signal.addEventListener('abort', listener) + }) + + if (this.isAborted) { + return + } + } + } +} + +export const createTick = async (interval: number): Promise => { + const tick = new Tick(interval) + + await tick.start() + + return tick +} diff --git a/packages/daemon/src/index.ts b/packages/daemon/src/index.ts index f0216dc7..f8ac71f4 100644 --- a/packages/daemon/src/index.ts +++ b/packages/daemon/src/index.ts @@ -1,60 +1,33 @@ +import setupCommon from './common/index.js' import { createLogger } from './logger.js' -import setupArgv from '@/modules/argv/index.js' -import setupBase from '@/modules/base/index.js' -import setupConfig from '@/modules/config/index.js' -import setupDownloader from '@/modules/downloader/index.js' +import parseArgv from './parse-argv.js' +import parseConfig from './parse-config.js' import setupFilesystem from '@/modules/filesystem/index.js' -import setupGroups from '@/modules/groups/index.js' -import setupNetwork from '@/modules/network/index.js' import setupRevisions from '@/modules/revisions/index.js' -import setupRPC from '@/modules/rpc/index.js' import setupScheduler from '@/modules/scheduler/index.js' -import setupSigint from '@/modules/sigint/index.js' -import setupSneakernet from '@/modules/sneakernet/index.js' -import setupTick from '@/modules/tick/index.js' const logger = createLogger('system') logger.info('starting...') -// Setup all the modules -const argv = await setupArgv() - -const sigint = await setupSigint() -const config = await setupConfig({ argv }) -const rpc = await setupRPC({ argv, sigint }) -const tick = await setupTick({ config, sigint }) -const base = await setupBase({ argv, config }) -const network = await setupNetwork({ sigint, config, base, rpc }) -const groups = await setupGroups({ sigint, base, network, rpc }) -const downloader = await setupDownloader({ config, base, network, rpc, tick, sigint }) - -await setupSneakernet({ base, groups, rpc, network }) -await setupScheduler({ base, groups, rpc }) +const argv = await parseArgv() +const config = await parseConfig(argv.config) -const filesystem = await setupFilesystem({ +// Setup all the modules +const components = await setupCommon({ config, - base, - network, - groups, - downloader, - tick, - rpc + socket: argv.socket, + key: argv.key }) -await setupRevisions({ - config, - base, - network, - groups, - filesystem, - downloader, - tick, - rpc -}) +await Promise.all([ + setupScheduler(components), + setupFilesystem(components), + setupRevisions(components) +]) process.on('SIGINT', () => { - sigint.interupt().catch(() => {}) + void components.stop() }) logger.info('started') diff --git a/packages/daemon/src/interface.ts b/packages/daemon/src/interface.ts index fa4641fa..bdcea6af 100644 --- a/packages/daemon/src/interface.ts +++ b/packages/daemon/src/interface.ts @@ -1,3 +1,4 @@ +import type { Components } from './common/interface.js' import type { Database, Keyvalue } from 'welo' export interface KeyvalueDB extends Database { @@ -7,21 +8,18 @@ export interface KeyvalueDB extends Database { export const MEMORY_MAGIC = ':memory:' export interface ModuleMethod< - Context extends Record = Record, - Components extends Record = Record + Context extends Record = Record > { - (context: Context, components: Components): void + (components: Components, context: Context): void } // Optional type to get around linter void issues. export type Optional = T extends undefined ? ReturnType<() => void> : T export interface Module< - Provides extends Record = Record, - Requires extends Record | undefined = undefined, - Init extends Record | undefined = undefined, + Context extends Record = Record > { - (components: Optional, init: Optional): Promise + (components: Components): Promise } export interface Pair { diff --git a/packages/daemon/src/modules/argv/index.ts b/packages/daemon/src/modules/argv/index.ts deleted file mode 100644 index 0e420d8f..00000000 --- a/packages/daemon/src/modules/argv/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import Path from 'path' -import { hideBin } from 'yargs/helpers' -import yargs from 'yargs/yargs' -import type { Module } from '@/interface.js' -import { projectPath } from '@/utils.js' - -export interface Provides extends Record { - socket: string - key: string - config: string -} - -const module: Module = async () => { - const argv = await yargs(hideBin(process.argv)) - .option({ - socket: { - alias: 's', - type: 'string', - default: '/tmp/server.socket' - } - }) - .option({ - key: { - alias: 'k', - type: 'string', - default: Path.join(projectPath, 'config/key.json') - } - }) - .option({ - config: { - alias: 'c', - type: 'string', - default: Path.join(projectPath, 'config/config.json') - } - }) - .parse() - - return { socket: argv.socket, key: argv.key, config: argv.config } -} - -export default module diff --git a/packages/daemon/src/modules/base/index.ts b/packages/daemon/src/modules/base/index.ts deleted file mode 100644 index 80231ca9..00000000 --- a/packages/daemon/src/modules/base/index.ts +++ /dev/null @@ -1,45 +0,0 @@ -import Path from 'path' -import { createKeyManager, type KeyManager } from '@organicdesign/db-key-manager' -import { MemoryBlockstore } from 'blockstore-core' -import { FsBlockstore } from 'blockstore-fs' -import { MemoryDatastore } from 'datastore-core' -import { FsDatastore } from 'datastore-fs' -import { z } from 'zod' -import type { Module } from '@/interface.js' -import type { Provides as Argv } from '@/modules/argv/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Blockstore } from 'interface-blockstore' -import type { Datastore } from 'interface-datastore' -import { isMemory } from '@/utils.js' - -const Config = z.object({ - storage: z.string().default(':memory:') -}) - -export interface Requires extends Record { - argv: Argv - config: ConfigModule -} - -export interface Provides extends Record { - datastore: Datastore - blockstore: Blockstore - keyManager: KeyManager -} - -const module: Module = async ({ argv, config }) => { - const c = config.get(Config) - const keyManager = await createKeyManager(Path.resolve(argv.key)) - - const datastore = isMemory(c.storage) - ? new MemoryDatastore() - : new FsDatastore(Path.join(c.storage, 'datastore')) - - const blockstore = isMemory(c.storage) - ? new MemoryBlockstore() - : new FsBlockstore(Path.join(c.storage, 'blockstore')) - - return { keyManager, datastore, blockstore } -} - -export default module diff --git a/packages/daemon/src/modules/config/index.ts b/packages/daemon/src/modules/config/index.ts deleted file mode 100644 index 586d160f..00000000 --- a/packages/daemon/src/modules/config/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -import fs from 'fs/promises' -import { z } from 'zod' -import type { Module } from '@/interface.js' -import type { Provides as Argv } from '@/modules/argv/index.js' - -export interface Requires extends Record { - argv: Argv -} - -export interface Provides extends Record { - config: Record - get (shape: T): z.infer -} - -const module: Module = async ({ argv }) => { - const raw = await fs.readFile(argv.config, { encoding: 'utf8' }) - const config = z.record(z.unknown()).parse(JSON.parse(raw)) - const get = (shape: T): z.infer => shape.parse(config) - - return { config, get } -} - -export default module diff --git a/packages/daemon/src/modules/downloader/downloader.ts b/packages/daemon/src/modules/downloader/downloader.ts deleted file mode 100644 index 14cdf65d..00000000 --- a/packages/daemon/src/modules/downloader/downloader.ts +++ /dev/null @@ -1,71 +0,0 @@ -import parallel from 'it-parallel' -import { pipe } from 'it-pipe' -import { type CID } from 'multiformats/cid' -import { collect } from 'streaming-iterables' -import { linearWeightTranslation } from './utils.js' -import { type Provides, logger } from './index.js' - -export default async (context: Provides, options?: { signal: AbortSignal }): Promise => { - const batchDownload = async function * (itr: AsyncIterable<[CID, number]>): AsyncGenerator<() => Promise<{ cid: CID, block: Uint8Array }>, void, undefined> { - for await (const [cid, priority] of itr) { - if (options?.signal.aborted === true) { - return - } - - const weight = Math.floor(linearWeightTranslation(priority / 100) * context.config.slots) + 1 - const downloaders = await context.pinManager.download(cid, { limit: weight }) - - yield * downloaders - } - } - - const catcher = async function * (itr: AsyncIterable): AsyncIterable { - try { - yield * itr - } catch (error) { - logger.warn('downloader threw: ', error) - yield * catcher(itr) - } - } - - const loop = async function * (): AsyncGenerator { - for (;;) { - if (options?.signal.aborted === true) { - return - } - - await new Promise(resolve => setTimeout(resolve, 100)) - yield - } - } - - const getPins = async function * (loop: AsyncIterable): AsyncGenerator<[CID, number]> { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const _ of loop) { - for await (const { value } of context.pinManager.getActive()) { - if (options?.signal.aborted === true) { - return - } - - yield [value.cid, value.priority] - } - } - } - - for (;;) { - if (options?.signal.aborted === true) { - return - } - - await pipe( - loop, - getPins, - batchDownload, - i => parallel(i, { concurrency: context.config.slots, ordered: false }), - i => catcher(i), - async i => collect(i) - ) - - await new Promise(resolve => setTimeout(resolve, 100)) - } -} diff --git a/packages/daemon/src/modules/downloader/index.ts b/packages/daemon/src/modules/downloader/index.ts deleted file mode 100644 index 5ce1c2af..00000000 --- a/packages/daemon/src/modules/downloader/index.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { z } from 'zod' -import getSpeeds from './commands/get-speeds.js' -import getStatus from './commands/get-status.js' -import setPriority from './commands/set-priority.js' -import download from './downloader.js' -import { PinManager } from './pin-manager.js' -import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' -import { createLogger } from '@/logger.js' -import { extendDatastore } from '@/utils.js' - -export const logger = createLogger('downloader') - -export const Config = z.object({ - slots: z.number().int().min(1).max(100).default(20) -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type Config = z.output - -export interface Requires extends Record { - base: Base - network: Network - rpc: RPC - config: ConfigModule - sigint: Sigint -} - -export interface Provides extends Record { - pinManager: PinManager - config: Config -} - -const module: Module = async (components) => { - const c = components.config.get(Config) - - const pinManager = new PinManager({ - pinManager: components.network.pinManager, - datastore: extendDatastore(components.base.datastore, 'pin-references') - }) - - const context = { pinManager, config: c } - - for (const setupCommand of [setPriority, getStatus, getSpeeds]) { - setupCommand(context, components) - } - - const controller = new AbortController() - - download(context, { signal: controller.signal }).catch(error => { - throw error - }) - - components.sigint.onInterupt(() => { controller.abort() }) - - return context -} - -export default module diff --git a/packages/daemon/src/modules/filesystem/commands/delete.ts b/packages/daemon/src/modules/filesystem/commands/delete.ts index 7397133a..2c835adb 100644 --- a/packages/daemon/src/modules/filesystem/commands/delete.ts +++ b/packages/daemon/src/modules/filesystem/commands/delete.ts @@ -1,12 +1,12 @@ import { Delete } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Delete.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, { uploads }) => { + net.rpc.addMethod(Delete.name, async (raw: unknown): Promise => { const params = Delete.Params.parse(raw) - const pairs = await context.uploads.add('delete', [CID.parse(params.group).bytes, params.path]) + const pairs = await uploads.add('delete', [CID.parse(params.group).bytes, params.path]) return pairs.map(p => ({ path: p.key, cid: p.value.cid.toString() })) }) diff --git a/packages/daemon/src/modules/filesystem/commands/edit.ts b/packages/daemon/src/modules/filesystem/commands/edit.ts index 88bad302..cf9e36bd 100644 --- a/packages/daemon/src/modules/filesystem/commands/edit.ts +++ b/packages/daemon/src/modules/filesystem/commands/edit.ts @@ -1,10 +1,10 @@ import { Edit } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import { type Provides, type Requires, logger } from '../index.js' +import { type Context, logger } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Edit.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, context) => { + net.rpc.addMethod(Edit.name, async (raw: unknown): Promise => { const params = Edit.Params.parse(raw) if (params.revisionStrategy !== null) { diff --git a/packages/daemon/src/modules/filesystem/commands/export.ts b/packages/daemon/src/modules/filesystem/commands/export.ts index 9252480f..73b0e994 100644 --- a/packages/daemon/src/modules/filesystem/commands/export.ts +++ b/packages/daemon/src/modules/filesystem/commands/export.ts @@ -2,11 +2,11 @@ import Path from 'path' import { Export } from '@organicdesign/db-rpc-interfaces' import { exporter } from '@organicdesign/db-utils' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, base }) => { - rpc.addMethod(Export.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, blockstore }, context) => { + net.rpc.addMethod(Export.name, async (raw: unknown): Promise => { const params = Export.Params.parse(raw) const fs = context.getFileSystem(CID.parse(params.group)) @@ -16,7 +16,7 @@ const command: ModuleMethod = (context, { rpc, base }) => { for await (const pair of fs.getDir(params.path)) { await exporter( - base.blockstore, + blockstore, Path.join(params.outPath, pair.key.toString().replace(params.path, '')), pair.value.cid ) diff --git a/packages/daemon/src/modules/filesystem/commands/import.ts b/packages/daemon/src/modules/filesystem/commands/import.ts index ca29240f..1900b08e 100644 --- a/packages/daemon/src/modules/filesystem/commands/import.ts +++ b/packages/daemon/src/modules/filesystem/commands/import.ts @@ -3,11 +3,11 @@ import { Import } from '@organicdesign/db-rpc-interfaces' import { selectChunker, importer, type ImporterConfig } from '@organicdesign/db-utils' import { BlackHoleBlockstore } from 'blockstore-core/black-hole' import { CID } from 'multiformats/cid' -import { type Provides, type Requires, logger } from '../index.js' +import { type Context, logger } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, network, base }) => { - rpc.addMethod(Import.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, blockstore, heliaPinManager }, context) => { + net.rpc.addMethod(Import.name, async (raw: unknown): Promise => { const params = Import.Params.parse(raw) const encrypt = Boolean(params.encrypt) @@ -24,12 +24,12 @@ const command: ModuleMethod = (context, { rpc, network, base logger.info('[add] importing %s', params.inPath) } - const store = params.onlyHash ? new BlackHoleBlockstore() : base.blockstore + const store = params.onlyHash ? new BlackHoleBlockstore() : blockstore const cids: Import.Return = [] for await (const r of importer(store, params.inPath, config)) { - await network.pinManager.pinLocal(r.cid) + await heliaPinManager.pinLocal(r.cid) logger.info('[add] imported %s', params.inPath) diff --git a/packages/daemon/src/modules/filesystem/commands/list.ts b/packages/daemon/src/modules/filesystem/commands/list.ts index cbbe627d..499f7fe5 100644 --- a/packages/daemon/src/modules/filesystem/commands/list.ts +++ b/packages/daemon/src/modules/filesystem/commands/list.ts @@ -1,15 +1,15 @@ import { List } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' import { toString as uint8arrayToString } from 'uint8arrays' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, groups }) => { - rpc.addMethod(List.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, groups }, context) => { + net.rpc.addMethod(List.name, async (raw: unknown): Promise => { const params = List.Params.parse(raw) const list: List.Return = [] - for (const { key: group } of groups.groups.all()) { + for (const { key: group } of groups.all()) { if (params.group != null && group !== params.group) { continue } diff --git a/packages/daemon/src/modules/filesystem/commands/read.ts b/packages/daemon/src/modules/filesystem/commands/read.ts index bc514f02..89d20a79 100644 --- a/packages/daemon/src/modules/filesystem/commands/read.ts +++ b/packages/daemon/src/modules/filesystem/commands/read.ts @@ -4,11 +4,11 @@ import { CID } from 'multiformats/cid' import { collect } from 'streaming-iterables' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, network }) => { - rpc.addMethod(Read.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, helia }, context) => { + net.rpc.addMethod(Read.name, async (raw: unknown): Promise => { const params = Read.Params.parse(raw) const fs = context.getFileSystem(CID.parse(params.group)) @@ -22,7 +22,7 @@ const command: ModuleMethod = (context, { rpc, network }) => throw new Error(`no such item: ${params.path}`) } - const ufs = unixfs(network.helia) + const ufs = unixfs(helia) return uint8ArrayToString(uint8ArrayConcat(await collect(ufs.cat(entry.cid, { offset: params.position, length: params.length })))) }) diff --git a/packages/daemon/src/modules/filesystem/commands/write.ts b/packages/daemon/src/modules/filesystem/commands/write.ts index afcff609..50f4b3c3 100644 --- a/packages/daemon/src/modules/filesystem/commands/write.ts +++ b/packages/daemon/src/modules/filesystem/commands/write.ts @@ -1,16 +1,16 @@ import { unixfs } from '@helia/unixfs' +import { CustomEvent } from '@libp2p/interface' import { Write } from '@organicdesign/db-rpc-interfaces' import all from 'it-all' import { CID } from 'multiformats/cid' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { FileSystemEvent } from '../events.js' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { Entry } from '../interface.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, network }) => { - rpc.addMethod(Write.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, helia, events }, context) => { + net.rpc.addMethod(Write.name, async (raw: unknown): Promise => { const params = Write.Params.parse(raw) const group = CID.parse(params.group) const fs = context.getFileSystem(CID.parse(params.group)) @@ -20,7 +20,7 @@ const command: ModuleMethod = (context, { rpc, network }) => } const entry: Partial = await fs.get(params.path) ?? {} - const ufs = unixfs(network.helia) + const ufs = unixfs(helia) const existingData = (entry.cid != null) ? uint8ArrayConcat(await all(ufs.cat(entry.cid))) : new Uint8Array() @@ -41,7 +41,7 @@ const command: ModuleMethod = (context, { rpc, network }) => const newEntry = await fs.put(params.path, newEntryParams) - context.events.dispatchEvent(new FileSystemEvent('file:added', group, params.path, newEntry)) + events.dispatchEvent(new CustomEvent('file:added', { detail: { group, path: params.path, entry: newEntry } })) return params.data.length }) diff --git a/packages/daemon/src/modules/filesystem/events.ts b/packages/daemon/src/modules/filesystem/events.ts deleted file mode 100644 index b8c99688..00000000 --- a/packages/daemon/src/modules/filesystem/events.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Event, type EventTarget } from 'ts-event-target' -import type { Entry } from './interface.js' -import type { CID } from 'multiformats/cid' - -export type EventTypes = 'file:added' - -export class FileSystemEvent extends Event { - readonly entry: Entry - readonly path: string - readonly group: CID - - constructor (type: EventTypes, group: CID, path: string, entry: Entry) { - super(type) - - this.entry = entry - this.path = path - this.group = group - } -} - -export type Events = EventTarget<[FileSystemEvent]> diff --git a/packages/daemon/src/modules/filesystem/index.ts b/packages/daemon/src/modules/filesystem/index.ts index 49ca8ab1..e95882e3 100644 --- a/packages/daemon/src/modules/filesystem/index.ts +++ b/packages/daemon/src/modules/filesystem/index.ts @@ -10,17 +10,9 @@ import write from './commands/write.js' import { type FileSystem } from './file-system.js' import setup from './setup.js' import syncGroups from './sync-groups.js' -import type { Events } from './events.js' import type { LocalSettings } from './local-settings.js' import type createUploadManager from './upload-operations.js' import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as Downloader } from '@/modules/downloader/index.js' -import type { Provides as Groups } from '@/modules/groups/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Tick } from '@/modules/tick/index.js' import type { CID } from 'multiformats/cid' import { createLogger } from '@/logger.js' @@ -33,26 +25,15 @@ export const Config = z.object({ // eslint-disable-next-line @typescript-eslint/no-redeclare export type Config = z.output -export interface Requires extends Record { - base: Base - network: Network - groups: Groups - downloader: Downloader - tick: Tick - rpc: RPC - config: ConfigModule -} - -export interface Provides extends Record { +export interface Context extends Record { uploads: Awaited> localSettings: LocalSettings config: Config getFileSystem (group: CID): FileSystem | null - events: Events } -const module: Module = async (components) => { - const config = components.config.get(Config) +const module: Module = async (components) => { + const config = components.parseConfig(Config) const context = await setup(components, config) for (const setupCommand of [ @@ -64,10 +45,10 @@ const module: Module = async (components) => { read, write ]) { - setupCommand(context, components) + setupCommand(components, context) } - components.tick.register(async () => syncGroups(components, context)) + components.tick.add(async () => syncGroups(components, context)) return context } diff --git a/packages/daemon/src/modules/filesystem/setup.ts b/packages/daemon/src/modules/filesystem/setup.ts index 2ea7750e..b6a1d26a 100644 --- a/packages/daemon/src/modules/filesystem/setup.ts +++ b/packages/daemon/src/modules/filesystem/setup.ts @@ -1,21 +1,20 @@ -import { EventTarget } from 'ts-event-target' import { FileSystem } from './file-system.js' import { LocalSettings } from './local-settings.js' import createUploadManager from './upload-operations.js' -import type { Events } from './events.js' -import type { Requires, Provides, Config } from './index.js' +import type { Context, Config } from './index.js' +import type { Components } from '@/common/interface.js' import type { CID } from 'multiformats/cid' import { extendDatastore } from '@/utils.js' -export default async (components: Requires, config: Config): Promise => { - const events: Events = new EventTarget() +export default async (components: Components, config: Config): Promise => { + const { groups, datastore, blockstore, welo } = components const localSettings = new LocalSettings({ - datastore: extendDatastore(components.base.datastore, 'references') + datastore: extendDatastore(datastore, 'references') }) const getFileSystem = (group: CID): FileSystem | null => { - const database = components.groups.groups.get(group) + const database = groups.get(group) if (database == null) { return null @@ -23,24 +22,22 @@ export default async (components: Requires, config: Config): Promise = return new FileSystem({ database, - blockstore: - components.base.blockstore, - id: components.groups.welo.identity.id, + blockstore, + id: welo.identity.id, localSettings }) } const uploads = await createUploadManager( - { getFileSystem, events }, + { getFileSystem }, components, - extendDatastore(components.base.datastore, 'upload-operations') + extendDatastore(datastore, 'upload-operations') ) return { localSettings, uploads, config, - getFileSystem, - events + getFileSystem } } diff --git a/packages/daemon/src/modules/filesystem/sync-groups.ts b/packages/daemon/src/modules/filesystem/sync-groups.ts index 07ab4085..00304311 100644 --- a/packages/daemon/src/modules/filesystem/sync-groups.ts +++ b/packages/daemon/src/modules/filesystem/sync-groups.ts @@ -1,11 +1,12 @@ import Path from 'path' import { DATA_KEY } from './interface.js' import { keyToPath } from './utils.js' -import { type Requires, type Provides, logger } from './index.js' +import { type Context, logger } from './index.js' +import type { Components } from '@/common/interface.js' -export default async ({ groups, downloader }: Requires, context: Provides): Promise => { - for (const { value: database } of groups.groups.all()) { - const tracker = groups.getTracker(database) +export default async ({ getTracker, groups, pinManager }: Components, context: Context): Promise => { + for (const { value: database } of groups.all()) { + const tracker = getTracker(database) for await (const { key } of tracker.process(`/${DATA_KEY}`)) { const path = keyToPath(key) @@ -22,11 +23,11 @@ export default async ({ groups, downloader }: Requires, context: Provides): Prom logger.info('[entry] syncing update:', fullKey) if (entry == null) { - await downloader.pinManager.remove(fullKey) + await pinManager.remove(fullKey) continue } - await downloader.pinManager.put(fullKey, { cid: entry.cid, priority: entry.priority }) + await pinManager.put(fullKey, { cid: entry.cid, priority: entry.priority }) } } } diff --git a/packages/daemon/src/modules/filesystem/upload-operations.ts b/packages/daemon/src/modules/filesystem/upload-operations.ts index 255badf5..674472a1 100644 --- a/packages/daemon/src/modules/filesystem/upload-operations.ts +++ b/packages/daemon/src/modules/filesystem/upload-operations.ts @@ -1,17 +1,18 @@ import Path from 'path' import { unixfs } from '@helia/unixfs' +import { CustomEvent } from '@libp2p/interface' import { type RevisionStrategies } from '@organicdesign/db-rpc-interfaces/zod' import all from 'it-all' import { CID } from 'multiformats/cid' import { take } from 'streaming-iterables' -import { FileSystemEvent } from './events.js' -import type { Requires, Provides } from './index.js' +import type { Context } from './index.js' import type { Entry } from './interface.js' +import type { Components } from '@/common/interface.js' import type { Pair } from '@/interface.js' import type { Datastore } from 'interface-datastore' import { OperationManager } from '@/operation-manager.js' -export default async (context: Pick, { network, downloader }: Requires, datastore: Datastore): Promise, { events, pinManager, helia }: Components, datastore: Datastore): Promise delete(groupData: Uint8Array, path: string): Promise>> }>> => { @@ -25,9 +26,9 @@ export default async (context: Pick, { net } const fullEntry = await fs.put(path, { ...entry, cid }) - await downloader.pinManager.put(path, { cid, priority: entry.priority }) + await pinManager.put(path, { cid, priority: entry.priority }) - context.events.dispatchEvent(new FileSystemEvent('file:added', group, path, fullEntry)) + events.dispatchEvent(new CustomEvent('file:added', { detail: { group, path, entry: fullEntry } })) } const om = new OperationManager(datastore, { @@ -41,7 +42,7 @@ export default async (context: Pick, { net } const parentPath = path.split('/').slice(0, -2).join('/') - const ufs = unixfs(network.helia) + const ufs = unixfs(helia) const cid = await ufs.addBytes(new Uint8Array([])) const pairs = await all(fs.getDir(path)) diff --git a/packages/daemon/src/modules/groups/commands/id.ts b/packages/daemon/src/modules/groups/commands/id.ts deleted file mode 100644 index ea21d97b..00000000 --- a/packages/daemon/src/modules/groups/commands/id.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { ID } from '@organicdesign/db-rpc-interfaces' -import { toString as uint8ArrayToString } from 'uint8arrays' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(ID.name, async (): Promise => { - return uint8ArrayToString(context.welo.identity.id, 'base58btc') - }) -} - -export default command diff --git a/packages/daemon/src/modules/groups/index.ts b/packages/daemon/src/modules/groups/index.ts deleted file mode 100644 index cdb8d99f..00000000 --- a/packages/daemon/src/modules/groups/index.ts +++ /dev/null @@ -1,53 +0,0 @@ -import createGroup from './commands/create-group.js' -import id from './commands/id.js' -import joinGroup from './commands/join-group.js' -import listGroups from './commands/list-groups.js' -import sync from './commands/sync.js' -import setupComponents from './setup.js' -import type { EntryTracker } from './entry-tracker.js' -import type { Groups } from './groups.js' -import type { Module, KeyvalueDB } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' -import type { Welo } from 'welo' -import { createLogger } from '@/logger.js' - -export const logger = createLogger('groups') - -export interface Requires extends Record { - base: Base - network: Network - rpc: RPC - sigint: Sigint -} - -export interface Provides extends Record { - welo: Welo - groups: Groups - getTracker(database: KeyvalueDB): EntryTracker -} - -const module: Module = async (components) => { - const context = await setupComponents(components) - - for (const setupCommand of [ - createGroup, - joinGroup, - listGroups, - sync, - id - ]) { - setupCommand(context, components) - } - - components.sigint.onInterupt(async () => { - await context.groups.stop() - await context.welo.stop() - }) - - return context -} - -export default module diff --git a/packages/daemon/src/modules/groups/setup.ts b/packages/daemon/src/modules/groups/setup.ts deleted file mode 100644 index f8025d2c..00000000 --- a/packages/daemon/src/modules/groups/setup.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { createWelo, pubsubReplicator, bootstrapReplicator } from 'welo' -import { EntryTracker } from './entry-tracker.js' -import { createGroups } from './groups.js' -import type { Requires, Provides } from './index.js' -import type { KeyvalueDB } from '@/interface.js' -import { extendDatastore } from '@/utils.js' - -export default async ({ base, network }: Requires): Promise => { - const welo = await createWelo({ - // @ts-expect-error Helia version mismatch here. - ipfs: network.helia, - replicators: [bootstrapReplicator(), pubsubReplicator()], - identity: await base.keyManager.getWeloIdentity() - }) - - const groups = await createGroups({ - datastore: extendDatastore(base.datastore, 'groups'), - welo - }) - - const getTracker = (database: KeyvalueDB): EntryTracker => new EntryTracker(database) - - return { - welo, - groups, - getTracker - } -} diff --git a/packages/daemon/src/modules/network/commands/addresses.ts b/packages/daemon/src/modules/network/commands/addresses.ts deleted file mode 100644 index 0bad66db..00000000 --- a/packages/daemon/src/modules/network/commands/addresses.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Addresses } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Addresses.name, async (): Promise => { - return context.libp2p.getMultiaddrs().map(a => a.toString()) - }) -} - -export default command diff --git a/packages/daemon/src/modules/network/commands/connections.ts b/packages/daemon/src/modules/network/commands/connections.ts deleted file mode 100644 index 34ba86dd..00000000 --- a/packages/daemon/src/modules/network/commands/connections.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Connections } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(Connections.name, async (): Promise => { - return context.libp2p.getConnections().map(c => c.remoteAddr.toString()) - }) -} - -export default command diff --git a/packages/daemon/src/modules/network/index.ts b/packages/daemon/src/modules/network/index.ts deleted file mode 100644 index 4966539d..00000000 --- a/packages/daemon/src/modules/network/index.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { z } from 'zod' -import addresses from './commands/addresses.js' -import connect from './commands/connect.js' -import connections from './commands/connections.js' -import countPeers from './commands/count-peers.js' -import setupComponents from './setup.js' -import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' -import type { Libp2p } from '@libp2p/interface' -import type PinManager from '@organicdesign/db-helia-pin-manager' -import type { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' -import type { Helia } from 'helia' -import { createLogger } from '@/logger.js' - -export const logger = createLogger('network') - -const Config = z.object({ - serverMode: z.boolean().default(false), - private: z.boolean().default(false), - bootstrap: z.array(z.string()).default([]), - addresses: z.array(z.string()).default([ - '/ip4/127.0.0.1/tcp/0', - '/ip4/127.0.0.1/tcp/0/ws' - ]) -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type Config = z.output - -export interface Requires extends Record { - base: Base - rpc: RPC - config: ConfigModule - sigint: Sigint -} - -export interface Provides extends Record { - libp2p: Libp2p - helia: Helia - pinManager: PinManager - config: Config - manualBlockBroker: ManualBlockBroker -} - -const module: Module = async (components) => { - const config = components.config.get(Config) - - const context = await setupComponents(components, config) - - for (const setupCommand of [addresses, connect, connections, countPeers]) { - setupCommand(context, components) - } - - components.sigint.onInterupt(async () => { - await context.helia.stop() - await context.libp2p.stop() - }) - - return context -} - -export default module diff --git a/packages/daemon/src/modules/network/setup.ts b/packages/daemon/src/modules/network/setup.ts deleted file mode 100644 index 0582cd11..00000000 --- a/packages/daemon/src/modules/network/setup.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { bitswap } from '@helia/block-brokers' -import PinManager from '@organicdesign/db-helia-pin-manager' -import { ManualBlockBroker } from '@organicdesign/db-manual-block-broker' -import { createHelia } from 'helia' -import createLibp2p from './libp2p.js' -import { type Requires, type Provides, type Config, logger } from './index.js' -import { extendDatastore } from '@/utils.js' - -export default async ({ base }: Requires, config: Config): Promise => { - const peerId = await base.keyManager.getPeerId() - const psk = base.keyManager.getPskKey() - - const libp2p = await createLibp2p({ - datastore: extendDatastore(base.datastore, 'datastore/libp2p'), - psk: config.private ? psk : undefined, - peerId, - ...config - }) - - const manualBlockBroker = new ManualBlockBroker() - - const helia = await createHelia({ - datastore: extendDatastore(base.datastore, 'helia/datastore'), - libp2p, - blockstore: base.blockstore, - blockBrokers: [bitswap(), () => manualBlockBroker] - }) - - const pinManager = new PinManager({ - helia, - datastore: extendDatastore(base.datastore, 'pinManager') - }) - - pinManager.events.addEventListener('downloads:added', ({ cid }) => { - logger.info(`[downloads] [+] ${cid}`) - }) - - pinManager.events.addEventListener('pins:added', ({ cid }) => { - logger.info(`[pins] [+] ${cid}`) - }) - - pinManager.events.addEventListener('pins:adding', ({ cid }) => { - logger.info(`[pins] [~] ${cid}`) - }) - - pinManager.events.addEventListener('pins:removed', ({ cid }) => { - logger.info(`[pins] [-] ${cid}`) - }) - - return { - libp2p, - helia, - pinManager, - config, - manualBlockBroker - } -} diff --git a/packages/daemon/src/modules/revisions/commands/export-revision.ts b/packages/daemon/src/modules/revisions/commands/export-revision.ts index 1eafc25f..ec28a88e 100644 --- a/packages/daemon/src/modules/revisions/commands/export-revision.ts +++ b/packages/daemon/src/modules/revisions/commands/export-revision.ts @@ -3,24 +3,22 @@ import { ExportRevision } from '@organicdesign/db-rpc-interfaces' import { exporter } from '@organicdesign/db-utils' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, base, filesystem }) => { - rpc.addMethod(ExportRevision.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, blockstore }, context) => { + net.rpc.addMethod(ExportRevision.name, async (raw: unknown): Promise => { const params = ExportRevision.Params.parse(raw) const group = CID.parse(params.group) const revisions = context.getRevisions(group) - const fs = filesystem.getFileSystem(group) - if (revisions == null || fs == null) { + if (revisions == null) { throw new Error('no such group') } const author = uint8ArrayFromString(params.author, 'base58btc') - for await (const pair of fs.getDir(params.path)) { - const path = pair.key.toString() + for await (const { path } of revisions.getAll(params.path)) { const outFile = Path.join(params.outPath, path.replace(params.path, '')) const revision = await revisions.get(path, author, params.sequence) @@ -29,7 +27,7 @@ const command: ModuleMethod = (context, { rpc, base, filesys } await exporter( - base.blockstore, + blockstore, outFile, revision.cid ) diff --git a/packages/daemon/src/modules/revisions/commands/list-revisions.ts b/packages/daemon/src/modules/revisions/commands/list-revisions.ts index a8a6f7f2..117efe53 100644 --- a/packages/daemon/src/modules/revisions/commands/list-revisions.ts +++ b/packages/daemon/src/modules/revisions/commands/list-revisions.ts @@ -1,11 +1,11 @@ import { ListRevisions } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' import { toString as uint8arrayToString } from 'uint8arrays/to-string' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(ListRevisions.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, context) => { + net.rpc.addMethod(ListRevisions.name, async (raw: unknown): Promise => { const params = ListRevisions.Params.parse(raw) const rs: ListRevisions.Return = [] const revisions = context.getRevisions(CID.parse(params.group)) diff --git a/packages/daemon/src/modules/revisions/commands/read-revision.ts b/packages/daemon/src/modules/revisions/commands/read-revision.ts index adc38295..e0135225 100644 --- a/packages/daemon/src/modules/revisions/commands/read-revision.ts +++ b/packages/daemon/src/modules/revisions/commands/read-revision.ts @@ -5,11 +5,11 @@ import { collect } from 'streaming-iterables' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc, network }) => { - rpc.addMethod(ReadRevision.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net, helia }, context) => { + net.rpc.addMethod(ReadRevision.name, async (raw: unknown): Promise => { const params = ReadRevision.Params.parse(raw) const group = CID.parse(params.group) const revisions = context.getRevisions(group) @@ -26,7 +26,7 @@ const command: ModuleMethod = (context, { rpc, network }) => throw new Error(`no such revision: ${params.path}, ${author}, ${params.sequence}`) } - const ufs = unixfs(network.helia) + const ufs = unixfs(helia) return uint8ArrayToString(uint8ArrayConcat(await collect(ufs.cat(entry.cid, { offset: params.position, length: params.length })))) }) diff --git a/packages/daemon/src/modules/revisions/index.ts b/packages/daemon/src/modules/revisions/index.ts index 534089a7..22e10821 100644 --- a/packages/daemon/src/modules/revisions/index.ts +++ b/packages/daemon/src/modules/revisions/index.ts @@ -7,14 +7,6 @@ import setup from './setup.js' import syncRevisions from './sync-revisions.js' import type { Revisions } from './revisions.js' import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as Downloader } from '@/modules/downloader/index.js' -import type { Provides as FileSystem } from '@/modules/filesystem/index.js' -import type { Provides as Groups } from '@/modules/groups/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import type { Provides as Tick } from '@/modules/tick/index.js' import type { CID } from 'multiformats/cid' import { createLogger } from '@/logger.js' @@ -27,29 +19,18 @@ export const Config = z.object({ // eslint-disable-next-line @typescript-eslint/no-redeclare export type Config = z.output -export interface Requires extends Record { - base: Base - network: Network - groups: Groups - filesystem: FileSystem - rpc: RPC - config: ConfigModule - downloader: Downloader - tick: Tick -} - -export interface Provides extends Record { +export interface Context extends Record { getRevisions (group: CID): Revisions | null } -const module: Module = async (components) => { +const module: Module = async (components) => { const context = await setup(components) for (const setupCommand of [listRevisions, exportRevision, readRevision]) { - setupCommand(context, components) + setupCommand(components, context) } - components.tick.register(async () => syncRevisions(components)) + components.tick.add(async () => syncRevisions(components)) return context } diff --git a/packages/daemon/src/modules/revisions/setup.ts b/packages/daemon/src/modules/revisions/setup.ts index 7d91f99a..e9b7c201 100644 --- a/packages/daemon/src/modules/revisions/setup.ts +++ b/packages/daemon/src/modules/revisions/setup.ts @@ -1,23 +1,31 @@ +import { CustomEvent } from '@libp2p/interface' import all from 'it-all' import { Revisions } from './revisions.js' import selectRevisions from './select-revisions.js' import { pathToKey } from './utils.js' -import { type Provides, type Requires, logger } from './index.js' +import { type Context, logger } from './index.js' +import type { Components } from '@/common/interface.js' import type { CID } from 'multiformats/cid' -export default async ({ filesystem, groups }: Requires): Promise => { +export default async ({ groups, welo, events }: Components): Promise => { const getRevisions = (group: CID): Revisions | null => { - const database = groups.groups.get(group) + const database = groups.get(group) if (database == null) { return null } - return new Revisions(database, groups.welo.identity.id) + return new Revisions(database, welo.identity.id) } - filesystem.events.addEventListener('file:added', ({ group, path, entry }) => { - (async () => { + events.addEventListener('file:added', (event) => { + if (!(event instanceof CustomEvent)) { + return + } + + const { detail: { group, path, entry } } = event + + ;(async () => { const revisions = getRevisions(group) if (revisions == null) { diff --git a/packages/daemon/src/modules/revisions/sync-revisions.ts b/packages/daemon/src/modules/revisions/sync-revisions.ts index 25e098c9..50db367f 100644 --- a/packages/daemon/src/modules/revisions/sync-revisions.ts +++ b/packages/daemon/src/modules/revisions/sync-revisions.ts @@ -2,11 +2,12 @@ import Path from 'path' import * as dagCbor from '@ipld/dag-cbor' import { VERSION_KEY, EncodedEntry } from './interface.js' import { decodeEntry } from './utils.js' -import { type Requires, logger } from './index.js' +import { logger } from './index.js' +import type { Components } from '@/common/interface.js' -export default async ({ groups, downloader }: Requires): Promise => { - for (const { value: database } of groups.groups.all()) { - const tracker = groups.getTracker(database) +export default async ({ groups, getTracker, pinManager }: Components): Promise => { + for (const { value: database } of groups.all()) { + const tracker = getTracker(database) const group = database.manifest.address.cid for await (const { key, value } of tracker.process(`/${VERSION_KEY}`)) { @@ -14,7 +15,7 @@ export default async ({ groups, downloader }: Requires): Promise => { const fullKey = Path.join('/', group.toString(), key) if (data == null) { - await downloader.pinManager.remove(fullKey) + await pinManager.remove(fullKey) continue } @@ -22,7 +23,7 @@ export default async ({ groups, downloader }: Requires): Promise => { logger.info('[entry] syncing update:', fullKey) - await downloader.pinManager.put(fullKey, { cid: entry.cid, priority: entry.priority }) + await pinManager.put(fullKey, { cid: entry.cid, priority: entry.priority }) } } } diff --git a/packages/daemon/src/modules/rpc/index.ts b/packages/daemon/src/modules/rpc/index.ts deleted file mode 100644 index cf346a19..00000000 --- a/packages/daemon/src/modules/rpc/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { createNetServer } from '@organicdesign/net-rpc' -import type { Module } from '@/interface.js' -import type { Provides as Argv } from '@/modules/argv/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' - -export interface Requires extends Record { - argv: Argv - sigint: Sigint -} - -export interface Provides extends Record { - addMethod (name: string, method: (arg: unknown) => any): void -} - -const module: Module = async (components) => { - const { rpc, close } = await createNetServer(components.argv.socket) - - const addMethod = (name: string, method: (arg: unknown) => any): void => { - rpc.addMethod(name, method) - } - - components.sigint.onInterupt(close) - - return { addMethod } -} - -export default module diff --git a/packages/daemon/src/modules/scheduler/commands/get-schedule.ts b/packages/daemon/src/modules/scheduler/commands/get-schedule.ts index c368ad0a..c78889bd 100644 --- a/packages/daemon/src/modules/scheduler/commands/get-schedule.ts +++ b/packages/daemon/src/modules/scheduler/commands/get-schedule.ts @@ -1,11 +1,11 @@ import { GetSchedule } from '@organicdesign/db-rpc-interfaces' import all from 'it-all' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(GetSchedule.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, context) => { + net.rpc.addMethod(GetSchedule.name, async (raw: unknown): Promise => { const params = GetSchedule.Params.parse(raw) const group = CID.parse(params.group) const schedule = context.getSchedule(group) diff --git a/packages/daemon/src/modules/scheduler/commands/put-schedule.ts b/packages/daemon/src/modules/scheduler/commands/put-schedule.ts index d1884a42..1a0e6266 100644 --- a/packages/daemon/src/modules/scheduler/commands/put-schedule.ts +++ b/packages/daemon/src/modules/scheduler/commands/put-schedule.ts @@ -1,10 +1,10 @@ import { PutSchedule } from '@organicdesign/db-rpc-interfaces' import { CID } from 'multiformats/cid' -import type { Provides, Requires } from '../index.js' +import type { Context } from '../index.js' import type { ModuleMethod } from '@/interface.js' -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(PutSchedule.name, async (raw: unknown): Promise => { +const command: ModuleMethod = ({ net }, context) => { + net.rpc.addMethod(PutSchedule.name, async (raw: unknown): Promise => { const params = PutSchedule.Params.parse(raw) const group = CID.parse(params.group) const schedule = context.getSchedule(group) diff --git a/packages/daemon/src/modules/scheduler/index.ts b/packages/daemon/src/modules/scheduler/index.ts index 6060b836..a3ed5768 100644 --- a/packages/daemon/src/modules/scheduler/index.ts +++ b/packages/daemon/src/modules/scheduler/index.ts @@ -4,9 +4,6 @@ import setupGetSchedule from './commands/get-schedule.js' import setupPutSchedule from './commands/put-schedule.js' import { Schedule } from './schedule.js' import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as Groups } from '@/modules/groups/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' import type { CID } from 'multiformats/cid' import { createLogger } from '@/logger.js' @@ -19,31 +16,25 @@ export const Config = z.object({ // eslint-disable-next-line @typescript-eslint/no-redeclare export type Config = z.output -export interface Requires extends Record { - groups: Groups - base: Base - rpc: RPC -} - -export interface Provides extends Record { +export interface Context extends Record { getSchedule(group: CID): Schedule | null } -const module: Module = async (components) => { +const module: Module = async (components) => { const getSchedule = (group: CID): Schedule | null => { - const database = components.groups.groups.get(group) + const database = components.groups.get(group) if (database == null) { return null } - return new Schedule(database, components.groups.welo.identity.id) + return new Schedule(database, components.welo.identity.id) } const context = { getSchedule } for (const setupCommand of [setupGetSchedule, setupPutSchedule]) { - setupCommand(context, components) + setupCommand(components, context) } return context diff --git a/packages/daemon/src/modules/sigint/index.ts b/packages/daemon/src/modules/sigint/index.ts deleted file mode 100644 index a2d188da..00000000 --- a/packages/daemon/src/modules/sigint/index.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { Module } from '@/interface.js' -import { createLogger } from '@/logger.js' - -export const logger = createLogger('sigint') - -export interface Provides extends Record { - onInterupt (method: () => unknown): void - interupt (): Promise -} - -const module: Module = async () => { - const methods: Array<() => unknown> = [] - - let exiting = false - - const interupt = async (): Promise => { - if (exiting) { - logger.warn('force exiting') - process.exit(1) - } - - exiting = true - - logger.info('cleaning up...') - - for (const method of methods) { - try { - await method() - } catch (error) { - logger.error(error) - } - } - - logger.info('exiting...') - } - - return { - onInterupt: (method) => methods.push(method), - interupt - } -} - -export default module diff --git a/packages/daemon/src/modules/sneakernet/commands/sneakernet-receive.ts b/packages/daemon/src/modules/sneakernet/commands/sneakernet-receive.ts deleted file mode 100644 index 30dd8041..00000000 --- a/packages/daemon/src/modules/sneakernet/commands/sneakernet-receive.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { SneakernetReveive } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(SneakernetReveive.name, async (raw: unknown): Promise => { - const params = SneakernetReveive.Params.parse(raw) - - await context.sneakernet.import(params.path) - - return null - }) -} - -export default command diff --git a/packages/daemon/src/modules/sneakernet/commands/sneakernet-send.ts b/packages/daemon/src/modules/sneakernet/commands/sneakernet-send.ts deleted file mode 100644 index 17ebfb21..00000000 --- a/packages/daemon/src/modules/sneakernet/commands/sneakernet-send.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { SneakernetSend } from '@organicdesign/db-rpc-interfaces' -import type { Provides, Requires } from '../index.js' -import type { ModuleMethod } from '@/interface.js' - -const command: ModuleMethod = (context, { rpc }) => { - rpc.addMethod(SneakernetSend.name, async (raw: unknown): Promise => { - const params = SneakernetSend.Params.parse(raw) - - await context.sneakernet.export(params.path, params.peers) - - return null - }) -} - -export default command diff --git a/packages/daemon/src/modules/sneakernet/index.ts b/packages/daemon/src/modules/sneakernet/index.ts deleted file mode 100644 index 9a50795d..00000000 --- a/packages/daemon/src/modules/sneakernet/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { RevisionStrategies } from '@organicdesign/db-rpc-interfaces/zod' -import { z } from 'zod' -import setupSneakernetReceive from './commands/sneakernet-receive.js' -import setupSneakernetSend from './commands/sneakernet-send.js' -import { Sneakernet } from './sneakernet.js' -import type { Module } from '@/interface.js' -import type { Provides as Base } from '@/modules/base/index.js' -import type { Provides as Groups } from '@/modules/groups/index.js' -import type { Provides as Network } from '@/modules/network/index.js' -import type { Provides as RPC } from '@/modules/rpc/index.js' -import { createLogger } from '@/logger.js' -import { extendDatastore } from '@/utils.js' - -export const logger = createLogger('sneakernet') - -export const Config = z.object({ - defaultRevisionStrategy: RevisionStrategies.default('all') -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type Config = z.output - -export interface Requires extends Record { - groups: Groups - base: Base - rpc: RPC - network: Network -} - -export interface Provides extends Record { - sneakernet: Sneakernet -} - -const module: Module = async (components) => { - // This module needs to be improved once the next gen Welo is released. - // It only transfers missing blocks but if we could workout difference between - // heads we could pre-emptively transfer missing data. - const datastore = extendDatastore(components.base.datastore, 'sneakernet') - const sneakernet = new Sneakernet(components, datastore) - const context = { sneakernet } - - for (const setupCommand of [setupSneakernetSend, setupSneakernetReceive]) { - setupCommand(context, components) - } - - return context -} - -export default module diff --git a/packages/daemon/src/modules/tick/index.ts b/packages/daemon/src/modules/tick/index.ts deleted file mode 100644 index 578126b6..00000000 --- a/packages/daemon/src/modules/tick/index.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { z } from 'zod' -import type { Module } from '@/interface.js' -import type { Provides as ConfigModule } from '@/modules/config/index.js' -import type { Provides as Sigint } from '@/modules/sigint/index.js' -import { createLogger } from '@/logger.js' - -export const logger = createLogger('tick') - -const Config = z.object({ - tickInterval: z.number().default(10 * 60) -}) - -// eslint-disable-next-line @typescript-eslint/no-redeclare -export type Config = z.output - -export interface Requires extends Record { - config: ConfigModule - sigint: Sigint -} - -export interface Provides extends Record { - config: Config - register (method: (...args: any[]) => any): void -} - -const module: Module = async (components) => { - const config = components.config.get(Config) - const methods: Array<(...args: any[]) => any> = [] - const register = (method: (...args: any[]) => any): void => { methods.push(method) } - - let stopping = false - - components.sigint.onInterupt(() => { - stopping = true - }) - - void (async () => { - for (;;) { - for (const method of methods) { - try { - await method() - } catch (error) { - logger.warn('method threw: ', error) - } - - if (stopping) { - return true - } - } - - const b = await new Promise(resolve => { - const timeout = setTimeout(() => { resolve(false) }, config.tickInterval) - - components.sigint.onInterupt(() => { - if (timeout != null) { - clearTimeout(timeout) - } - - resolve(true) - }) - }) - - if (b) { - return - } - } - })() - - return { config, register } -} - -export default module diff --git a/packages/daemon/src/parse-argv.ts b/packages/daemon/src/parse-argv.ts new file mode 100644 index 00000000..d26fb602 --- /dev/null +++ b/packages/daemon/src/parse-argv.ts @@ -0,0 +1,37 @@ +import Path from 'path' +import { hideBin } from 'yargs/helpers' +import yargs from 'yargs/yargs' + +export default async (): Promise<{ + socket: string + key?: string + config?: string +}> => { + const argv = await yargs(hideBin(process.argv)) + .option({ + socket: { + alias: 's', + type: 'string', + default: '/tmp/server.socket' + } + }) + .option({ + key: { + alias: 'k', + type: 'string' + } + }) + .option({ + config: { + alias: 'c', + type: 'string' + } + }) + .parse() + + return { + socket: Path.resolve(argv.socket), + key: argv.key != null ? Path.resolve(argv.key) : undefined, + config: argv.config != null ? Path.resolve(argv.config) : undefined + } +} diff --git a/packages/daemon/src/parse-config.ts b/packages/daemon/src/parse-config.ts new file mode 100644 index 00000000..232edebe --- /dev/null +++ b/packages/daemon/src/parse-config.ts @@ -0,0 +1,9 @@ +import fs from 'fs/promises' +import { z } from 'zod' + +export default async (path?: string): Promise> => { + const raw = path != null ? await fs.readFile(path, { encoding: 'utf8' }) : '{}' + const config = z.record(z.unknown()).parse(JSON.parse(raw)) + + return config +} diff --git a/packages/daemon/test/modules/argv.spec.ts b/packages/daemon/test/modules/argv.spec.ts index 483f2fa8..b31ab2e0 100644 --- a/packages/daemon/test/modules/argv.spec.ts +++ b/packages/daemon/test/modules/argv.spec.ts @@ -1,15 +1,13 @@ import assert from 'assert/strict' -import Path from 'path' -import argv from '../../src/modules/argv/index.js' -import { projectPath } from '@/utils.js' +import parseArgv from '../../src/parse-argv.js' describe('argv', () => { it('returns defaults for every argv parameter', async () => { - const m = await argv() + const argv = await parseArgv() - assert.equal(m.key, Path.join(projectPath, 'config/key.json')) - assert.equal(m.config, Path.join(projectPath, 'config/config.json')) - assert.equal(m.socket, '/tmp/server.socket') + assert.equal(argv.key, undefined) + assert.equal(argv.config, undefined) + assert.equal(argv.socket, '/tmp/server.socket') }) it('returns the value for every argv parameter', async () => { @@ -24,10 +22,10 @@ describe('argv', () => { process.argv.push('--socket') process.argv.push(socket) - const m = await argv() + const argv = await parseArgv() - assert.equal(m.key, key) - assert.equal(m.config, config) - assert.equal(m.socket, socket) + assert.equal(argv.key, key) + assert.equal(argv.socket, socket) + assert.equal(argv.config, config) }) }) diff --git a/packages/daemon/test/modules/base.spec.ts b/packages/daemon/test/modules/base.spec.ts index 81f9c180..dbccc8b5 100644 --- a/packages/daemon/test/modules/base.spec.ts +++ b/packages/daemon/test/modules/base.spec.ts @@ -8,26 +8,21 @@ import { FsDatastore } from 'datastore-fs' import { Key } from 'interface-datastore' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays' -import base from '../../src/modules/base/index.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockConfig from './mock-config.js' -import type { Provides as Argv } from '../../src/modules/argv/index.js' +import setup from '@/common/index.js' import { extendDatastore } from '@/utils.js' const parseStr = (data: string): Uint8Array => uint8ArrayFromString(data, 'base64') const testPath = mkTestPath('base') describe('base', () => { - let argv: Argv + const keyPath = Path.join(testPath, 'key.json') + const socket = Path.join(testPath, 'server.socket') before(async () => { - argv = mockArgv(testPath) - - await fs.mkdir(Path.join(argv.key, '..'), { recursive: true }) await fs.mkdir(testPath, { recursive: true }) - await fs.writeFile(argv.key, JSON.stringify({ + await fs.writeFile(keyPath, JSON.stringify({ key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' })) @@ -38,67 +33,61 @@ describe('base', () => { }) it('returns the key manager', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) + const components = await setup({ key: keyPath, socket }) assert.deepEqual( - new Uint8Array(m.keyManager.aesKey), + new Uint8Array(components.keyManager.aesKey), parseStr('knUGn6uUeQcoxfM1qAtg3F/Njm4bp+GcZK257NZ5AtE') ) assert.deepEqual( - new Uint8Array(m.keyManager.hmacKey), + new Uint8Array(components.keyManager.hmacKey), parseStr('KZkJfNz3bRrn6XHvWYtD4+dXvmhdT4TBhBIkWn8y3jY') ) assert.deepEqual( - (await m.keyManager.getWeloIdentity()).id, + (await components.keyManager.getWeloIdentity()).id, parseStr('CAISIQPSvjmKINqJY5SA/3c+kadFmIsHeTXtTJYlrooZ53DTUg') ) assert.deepEqual( - (await m.keyManager.getPeerId()).toBytes(), + (await components.keyManager.getPeerId()).toBytes(), parseStr('ACUIAhIhA5m2/DfXxqi0i+fyYixRaWGirDEVemxUEv8WMZPwFPZB') ) assert.deepEqual( - m.keyManager.getPskKey(), + components.keyManager.getPskKey(), parseStr('L2tleS9zd2FybS9wc2svMS4wLjAvCi9iYXNlMTYvCjU2ZDNjMTgyODJmMWYxYjFiM2UwNGU0MGRkNWQ4YmY0NGNhZmE4YmM5YzliYzdjNTc3MTZhNzc2NmZhMmM1NTA') ) + + await components.stop() }) it('uses memory blockstore when memory is specified', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) + const components = await setup({ socket, config: { storage: ':memory:' } }) - assert(m.blockstore instanceof MemoryBlockstore) + assert(components.blockstore instanceof MemoryBlockstore) + + await components.stop() }) it('uses memory datastore when memory is specified', async () => { - const m = await base({ - config: mockConfig({ storage: ':memory:' }), - argv - }) + const components = await setup({ socket, config: { storage: ':memory:' } }) + + assert(components.datastore instanceof MemoryDatastore) - assert(m.datastore instanceof MemoryDatastore) + await components.stop() }) it('uses fs blockstore when a path is specified', async () => { const blockstorePath = Path.join(testPath, 'blockstore') const testData = uint8ArrayFromString('test') - const m = await base({ - config: mockConfig({ storage: testPath }), - argv - }) + const components = await setup({ socket, config: { storage: testPath } }) - assert(m.blockstore instanceof FsBlockstore) + assert(components.blockstore instanceof FsBlockstore) - await m.blockstore.put(CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ'), testData) + await components.blockstore.put(CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ'), testData) const out = await fs.readdir(blockstorePath, { recursive: true }) @@ -110,21 +99,22 @@ describe('base', () => { const blockData = await fs.readFile(Path.join(blockstorePath, '7I/BCIQLASSX2QHMUE4IBHYTTJ3LCICEGM6DOQBZDSN7DTU5RYQ2PEQQX7I.data')) assert.deepEqual(new Uint8Array(blockData), testData) + + await components.stop() }) it('uses fs datastore when a path is specified', async () => { - const datastorePath = Path.join(testPath, 'datastore') + const datastorePath = Path.join(testPath, 'datastore', 'test') + + const components = await setup({ socket, config: { storage: testPath } }) - const m = await base({ - config: mockConfig({ storage: testPath }), - argv - }) + assert(components.datastore instanceof FsDatastore) - assert(m.datastore instanceof FsDatastore) + const datastore = extendDatastore(components.datastore, 'test') - await m.datastore.put(new Key('key'), uint8ArrayFromString('value')) + await datastore.put(new Key('key'), uint8ArrayFromString('value')) - const subDatastore = extendDatastore(extendDatastore(m.datastore, 'a'), 'b/c') + const subDatastore = extendDatastore(extendDatastore(datastore, 'a'), 'b/c') await subDatastore.put(new Key('d/e'), uint8ArrayFromString('test')) const out = await fs.readdir(datastorePath, { recursive: true }) @@ -143,5 +133,7 @@ describe('base', () => { assert.deepEqual(new Uint8Array(data1), uint8ArrayFromString('value')) assert.deepEqual(new Uint8Array(data2), uint8ArrayFromString('test')) + + await components.stop() }) }) diff --git a/packages/daemon/test/modules/config.spec.ts b/packages/daemon/test/modules/config.spec.ts index 26c93b57..df1fcb81 100644 --- a/packages/daemon/test/modules/config.spec.ts +++ b/packages/daemon/test/modules/config.spec.ts @@ -2,8 +2,9 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' import { z } from 'zod' -import config from '../../src/modules/config/index.js' +import parseConfig from '../../src/parse-config.js' import { mkTestPath } from '../utils/paths.js' +import setup from '@/common/index.js' const testPath = mkTestPath('config') const configPath = Path.join(testPath, 'config.json') @@ -28,27 +29,25 @@ after(async () => { }) describe('config', () => { - it('returns parsed config from the file', async () => { - const m = await config({ - argv: { config: configPath, key: '', socket: '' } - }) + it('gets config from file', async () => { + const config = await parseConfig(configPath) - assert.deepEqual(m.config, configData) + assert.deepEqual(config, configData) }) - it('gets config from schema', async () => { - const m = await config({ - argv: { config: configPath, key: '', socket: '' } - }) + it('parses config from schema', async () => { + const components = await setup({ config: configData, socket: Path.join(testPath, 'server.socket') }) assert.deepEqual( - m.get(z.object({ bootstrap: z.array(z.string()) })), + components.parseConfig(z.object({ bootstrap: z.array(z.string()) })), { bootstrap: configData.bootstrap } ) assert.deepEqual( - m.get(z.object({ tickInterval: z.number(), private: z.boolean() })), + components.parseConfig(z.object({ tickInterval: z.number(), private: z.boolean() })), { tickInterval: configData.tickInterval, private: configData.private } ) + + await components.stop() }) }) diff --git a/packages/daemon/test/modules/downloader.spec.ts b/packages/daemon/test/modules/downloader.spec.ts index 5c459870..fa5c4246 100644 --- a/packages/daemon/test/modules/downloader.spec.ts +++ b/packages/daemon/test/modules/downloader.spec.ts @@ -1,65 +1,22 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' -import { KeyManager } from '@organicdesign/db-key-manager' import { createNetClient } from '@organicdesign/net-rpc' import { MemoryBlockstore } from 'blockstore-core' import { CID } from 'multiformats/cid' -import createDownloader from '../../src/modules/downloader/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { - Requires as DownloaderComponents, - Provides as DownloaderProvides -} from '../../src/modules/downloader/index.js' +import type { Components } from '@/common/interface.js' +import setup from '@/common/index.js' describe('downloader', () => { const testPath = mkTestPath('groups') - const create = async (name?: string): Promise & { - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - downloader: DownloaderProvides - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) + const create = async (): Promise<{ components: Components, socket: string }> => { + const socket = Path.join(testPath, `${Math.random()}.socket`) + const components = await setup({ socket }) - return { - argv, - config, - sigint, - rpc, - base, - network, - downloader - } + return { components, socket } } before(async () => { @@ -70,48 +27,40 @@ describe('downloader', () => { await fs.rm(testPath, { recursive: true }) }) - it('has 20 slots by default', async () => { - const { downloader: m, sigint } = await create() - - assert.equal(m.config.slots, 20) - - await sigint.interupt() - }) - it('rpc - set priority updates local priority', async () => { const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' const path = '/test.txt' const priority = 50 - const { downloader: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const key = Path.join('/', group, path) - await m.pinManager.put(key, { priority: 1, cid: CID.parse(group) }) + await components.pinManager.put(key, { priority: 1, cid: CID.parse(group) }) const response = await client.rpc.request('set-priority', { group, path, priority }) assert.equal(response, null) - const pinData = await m.pinManager.get(key) + const pinData = await components.pinManager.get(key) assert(pinData != null) assert.equal(pinData.priority, priority) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - get speed', async () => { - const { downloader: m, network, sigint, argv } = await create() + const { components, socket } = await create() const blockstore = new MemoryBlockstore() const dag = await createDag({ blockstore }, 2, 2) const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' const path = '/test.txt' const range = 500 - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const key = Path.join('/', group, path) - await m.pinManager.put(key, { priority: 1, cid: dag[0] }) + await components.pinManager.put(key, { priority: 1, cid: dag[0] }) const speed1 = await client.rpc.request('get-speeds', { cids: [dag[0].toString()], @@ -122,7 +71,7 @@ describe('downloader', () => { const value = await blockstore.get(dag[0]) - await network.helia.blockstore.put(dag[0], value) + await components.helia.blockstore.put(dag[0], value) await new Promise(resolve => setTimeout(resolve, range / 2)) const speed2 = await client.rpc.request('get-speeds', { @@ -140,8 +89,8 @@ describe('downloader', () => { ]) await Promise.all([ - network.helia.blockstore.put(dag[1], values[0]), - network.helia.blockstore.put(dag[4], values[1]) + components.helia.blockstore.put(dag[1], values[0]), + components.helia.blockstore.put(dag[4], values[1]) ]) await new Promise(resolve => setTimeout(resolve, range / 2)) @@ -157,16 +106,16 @@ describe('downloader', () => { }]) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - get status', async () => { - const { downloader: m, network, sigint, argv } = await create() + const { components, socket } = await create() const blockstore = new MemoryBlockstore() const dag = await createDag({ blockstore }, 2, 2) const group = 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' const path = '/test.txt' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const key = Path.join('/', group, path) const status1 = await client.rpc.request('get-status', { @@ -180,7 +129,7 @@ describe('downloader', () => { state: 'NOTFOUND' }]) - await m.pinManager.put(key, { priority: 1, cid: dag[0] }) + await components.pinManager.put(key, { priority: 1, cid: dag[0] }) const status2 = await client.rpc.request('get-status', { cids: [dag[0].toString()] @@ -195,7 +144,7 @@ describe('downloader', () => { const value = await blockstore.get(dag[0]) - await network.helia.blockstore.put(dag[0], value) + await components.helia.blockstore.put(dag[0], value) await new Promise(resolve => setTimeout(resolve, 100)) const status3 = await client.rpc.request('get-status', { @@ -212,7 +161,7 @@ describe('downloader', () => { const values = await Promise.all(dag.map(async cid => { const value = await blockstore.get(cid) - await network.helia.blockstore.put(cid, value) + await components.helia.blockstore.put(cid, value) return value.length })) @@ -231,6 +180,6 @@ describe('downloader', () => { }]) client.close() - await sigint.interupt() + await components.stop() }) }) diff --git a/packages/daemon/test/modules/filesystem.spec.ts b/packages/daemon/test/modules/filesystem.spec.ts index 2c1d8896..18a58032 100644 --- a/packages/daemon/test/modules/filesystem.spec.ts +++ b/packages/daemon/test/modules/filesystem.spec.ts @@ -2,7 +2,6 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' import { unixfs } from '@helia/unixfs' -import { KeyManager } from '@organicdesign/db-key-manager' import * as testData from '@organicdesign/db-test-utils' import { importer } from '@organicdesign/db-utils' import { createNetClient } from '@organicdesign/net-rpc' @@ -11,88 +10,27 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createDownloader from '../../src/modules/downloader/index.js' -import createFilesystem from '../../src/modules/filesystem/index.js' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import createTick from '../../src/modules/tick/index.js' +import setupFilesystem from '../../src/modules/filesystem/index.js' import { createGroup } from '../utils/create-group.js' import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { Provides as FSProvides } from '../../src/modules/filesystem/index.js' +import type { Context as FilesystemContext } from '../../src/modules/filesystem/index.js' +import type { Components } from '@/common/interface.js' +import setup from '@/common/index.js' describe('filesystem', () => { const testPath = mkTestPath('filesystem') - const create = async (name?: string): Promise<{ - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: Awaited> - filesystem: FSProvides - sigint: Awaited> - tick: Awaited> + const create = async (): Promise<{ + filesystem: FilesystemContext + components: Components + socket: string }> => { - const path = name == null ? testPath : Path.join(testPath, name) + const socket = Path.join(testPath, `${Math.random()}.socket`) + const components = await setup({ socket }) + const filesystem = await setupFilesystem(components) - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) - - const tick = await createTick({ config, sigint }) - - const filesystem = await createFilesystem({ - sigint, - base, - rpc, - network, - groups, - downloader, - tick, - config - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups, - filesystem, - tick - } + return { filesystem, components, socket } } before(async () => { @@ -104,44 +42,44 @@ describe('filesystem', () => { }) it('uses all as the default revision strategy', async () => { - const { filesystem: m, sigint } = await create() + const { filesystem, components } = await create() - assert.equal(m.config.defaultRevisionStrategy, 'all') + assert.equal(filesystem.config.defaultRevisionStrategy, 'all') - await sigint.interupt() + await components.stop() }) it('returns null when wraping a group that doesn\'t exist', async () => { - const { filesystem: m, sigint } = await create() - const fs = m.getFileSystem(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) + const { filesystem, components } = await create() + const fs = filesystem.getFileSystem(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) assert.equal(fs, null) - await sigint.interupt() + await components.stop() }) it('wraps a group in a filesystem', async () => { - const { filesystem: m, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components } = await create() + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) assert.notEqual(fs, null) - await sigint.interupt() + await components.stop() }) it('uploads show in filesystem', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components } = await create() + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const path = '/test' assert(fs != null) const before = Date.now() - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -149,11 +87,11 @@ describe('filesystem', () => { }]) const entry = await fs.get(path) - const values = await Promise.all(dag.map(async d => network.helia.blockstore.get(d))) + const values = await Promise.all(dag.map(async d => components.helia.blockstore.get(d))) const size = values.reduce((a, c) => a + c.length, 0) assert(entry != null) - assert.deepEqual(entry.author, groups.welo.identity.id) + assert.deepEqual(entry.author, components.welo.identity.id) assert.equal(entry.blocks, dag.length) assert.deepEqual(entry.cid, dag[0]) assert.equal(entry.encrypted, false) @@ -164,22 +102,22 @@ describe('filesystem', () => { assert(entry.timestamp >= before) assert(entry.timestamp <= Date.now()) - await sigint.interupt() + await components.stop() }) it('emits the file:added event when the FS was written to', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components } = await create() + const group = await createGroup(components, 'test') + const dag = await createDag(components.helia, 2, 2) const path = '/test' const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - m.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -188,26 +126,26 @@ describe('filesystem', () => { await promise - await sigint.interupt() + await components.stop() }) it('local settings change FS output', async () => { - const { filesystem: m, groups, network, sigint } = await create() - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components } = await create() + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const path = '/test' assert(fs != null) - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, priority: 1 }]) - await m.localSettings.set(group, path, { + await filesystem.localSettings.set(group, path, { priority: 100 }) @@ -216,20 +154,20 @@ describe('filesystem', () => { assert(entry != null) assert.equal(entry.priority, 100) - await sigint.interupt() + await components.stop() }) it('rpc - delete (file)', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const path = '/test' assert(fs != null) - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -245,22 +183,22 @@ describe('filesystem', () => { assert.equal(entry, null) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - delete (directory)', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const rootPath = '/test' const paths = [`${rootPath}/file1`, `${rootPath}/file2`, `${rootPath}/sub/file3`] assert(fs != null) await Promise.all(paths.map(async path => - m.uploads.add('put', [group.bytes, path, { + filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -277,14 +215,14 @@ describe('filesystem', () => { assert.deepEqual(entries, []) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - edit priority', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const path = '/test' const priority = 50 @@ -294,28 +232,28 @@ describe('filesystem', () => { assert.equal(response, null) - const localSettings = await m.localSettings.get(group, path) + const localSettings = await filesystem.localSettings.get(group, path) assert.equal(localSettings.priority, priority) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - export (file)', async () => { - const { filesystem: m, network, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const rootPath = '/test' const outPath = Path.join(testPath, 'export-file') assert(fs != null) await Promise.all(testData.data.map(async data => { - const result = await all(importer(network.helia.blockstore, data.path)) + const result = await all(importer(components.helia.blockstore, data.path)) - await m.uploads.add('put', [group.bytes, data.generatePath(rootPath), { + await filesystem.uploads.add('put', [group.bytes, data.generatePath(rootPath), { cid: result[0].cid.bytes, encrypted: false, revisionStrategy: 'all', @@ -339,23 +277,23 @@ describe('filesystem', () => { } client.close() - await sigint.interupt() + await components.stop() }) it('rpc - export (directory)', async () => { - const { filesystem: m, network, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const rootPath = '/test' const outPath = Path.join(testPath, 'export-directory') assert(fs != null) await Promise.all(testData.data.map(async data => { - const result = await all(importer(network.helia.blockstore, data.path)) + const result = await all(importer(components.helia.blockstore, data.path)) - await m.uploads.add('put', [group.bytes, data.generatePath(rootPath), { + await filesystem.uploads.add('put', [group.bytes, data.generatePath(rootPath), { cid: result[0].cid.bytes, encrypted: false, revisionStrategy: 'all', @@ -379,14 +317,14 @@ describe('filesystem', () => { } client.close() - await sigint.interupt() + await components.stop() }) it('rpc - import (file)', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const rootPath = '/test' assert(fs != null) @@ -413,14 +351,14 @@ describe('filesystem', () => { })) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - import (directory)', async () => { - const { filesystem: m, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const rootPath = '/test' assert(fs != null) @@ -456,15 +394,15 @@ describe('filesystem', () => { } client.close() - await sigint.interupt() + await components.stop() }) it('rpc - list', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) + const dag = await createDag(components.helia, 2, 2) const rootPath = '/test' const paths = [`${rootPath}/file1`, `${rootPath}/file2`, `${rootPath}/sub/file3`] @@ -473,7 +411,7 @@ describe('filesystem', () => { const before = Date.now() await Promise.all(paths.map(async path => - m.uploads.add('put', [group.bytes, path, { + filesystem.uploads.add('put', [group.bytes, path, { cid: dag[0].bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -485,12 +423,12 @@ describe('filesystem', () => { assert(Array.isArray(response)) - const values = await Promise.all(dag.map(async d => network.helia.blockstore.get(d))) + const values = await Promise.all(dag.map(async d => components.helia.blockstore.get(d))) const size = values.reduce((a, c) => a + c.length, 0) for (const entry of response) { assert(entry != null) - assert.equal(entry.author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(entry.author, uint8ArrayToString(components.welo.identity.id, 'base58btc')) assert.equal(entry.blocks, dag.length) assert.deepEqual(entry.cid, dag[0].toString()) assert.equal(entry.encrypted, false) @@ -502,20 +440,20 @@ describe('filesystem', () => { } client.close() - await sigint.interupt() + await components.stop() }) it('rpc - read', async () => { - const { filesystem: m, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const ufs = unixfs(network.helia) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const ufs = unixfs(components.helia) const path = '/test' const data = 'test-data' const cid = await ufs.addBytes(uint8ArrayFromString(data)) - await m.uploads.add('put', [group.bytes, path, { + await filesystem.uploads.add('put', [group.bytes, path, { cid: cid.bytes, encrypted: false, revisionStrategy: 'all' as const, @@ -539,14 +477,14 @@ describe('filesystem', () => { assert.deepEqual(read4, data.slice(1, 3 + 1)) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - write', async () => { - const { filesystem: m, base, groups, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const fs = m.getFileSystem(group) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const fs = filesystem.getFileSystem(group) const ufs = unixfs({ blockstore: new MemoryBlockstore() }) const path = '/test' const data = 'test-data' @@ -563,7 +501,7 @@ describe('filesystem', () => { const entry1 = await fs.get(path) assert(entry1 != null) - assert.deepEqual(entry1.author, groups.welo.identity.id) + assert.deepEqual(entry1.author, components.welo.identity.id) assert.equal(entry1.blocks, 1) assert.deepEqual(entry1.cid, cid) assert.equal(entry1.encrypted, false) @@ -582,7 +520,7 @@ describe('filesystem', () => { assert(entry2 != null) - const value2 = await base.blockstore.get(entry2.cid) + const value2 = await components.blockstore.get(entry2.cid) assert.deepEqual(value2, uint8ArrayFromString(newData2)) @@ -595,7 +533,7 @@ describe('filesystem', () => { assert(entry3 != null) - const value3 = await base.blockstore.get(entry3.cid) + const value3 = await components.blockstore.get(entry3.cid) assert.deepEqual(value3, uint8ArrayFromString('test-data-long')) @@ -608,11 +546,11 @@ describe('filesystem', () => { assert(entry4 != null) - const value4 = await base.blockstore.get(entry4.cid) + const value4 = await components.blockstore.get(entry4.cid) assert.deepEqual(value4, uint8ArrayFromString('test-long-long')) client.close() - await sigint.interupt() + await components.stop() }) }) diff --git a/packages/daemon/test/modules/groups.spec.ts b/packages/daemon/test/modules/groups.spec.ts index f94d84f9..d1cdaeae 100644 --- a/packages/daemon/test/modules/groups.spec.ts +++ b/packages/daemon/test/modules/groups.spec.ts @@ -1,88 +1,20 @@ import assert from 'assert/strict' import fs from 'fs/promises' import Path from 'path' -import { KeyManager, parseKeyData } from '@organicdesign/db-key-manager' import { createNetClient } from '@organicdesign/net-rpc' import * as cborg from 'cborg' import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' +import { createGroup } from '../utils/create-group.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { - Requires as GroupsComponents, - Provides as GroupsProvides -} from '../../src/modules/groups/index.js' +import type { Components } from '@/common/interface.js' +import setup from '@/common/index.js' describe('groups', () => { const testPath = mkTestPath('groups') - const mkGroup = async (m: GroupsProvides, name: string, peers: Uint8Array[] = []): Promise => { - const manifest = await m.welo.determine({ - name, - meta: { type: 'group' }, - access: { - protocol: '/hldb/access/static', - config: { write: [m.welo.identity.id, ...peers] } - } - }) - - await m.groups.add(manifest) - - return manifest.address.cid - } - - const create = async (name?: string): Promise & { - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: GroupsProvides - }> => { - const path = name == null ? testPath : Path.join(testPath, name) - - const keyManager = name == null - ? undefined - : new KeyManager(parseKeyData({ - key: 'DpGbLiAX4wK4HHtG3DQb8cA6FG2ibv93X4ooZJ5LmMJJ-12FmenN8dbWysuYnzEHzmEF1hod4RGK8NfKFu1SEZ7XM', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n023330a98e30315e2233d4a31a6dc65d741a89f7ce6248e7de40c73995d23157' - })) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups - } - } - before(async () => { await fs.mkdir(testPath, { recursive: true }) }) @@ -91,25 +23,32 @@ describe('groups', () => { await fs.rm(testPath, { recursive: true }) }) + const create = async (): Promise<{ components: Components, socket: string }> => { + const socket = Path.join(testPath, `${Math.random()}.socket`) + const components = await setup({ socket }) + + return { components, socket } + } + it('creates a group', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') + const { components } = await create() + const group = await createGroup(components, 'test') assert(group) - await sigint.interupt() + await components.stop() }) it('tracker puts and checks a group\'s content', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) + const { components } = await create() + const group = await createGroup(components, 'test') + const database = components.groups.get(group) if (database == null) { throw new Error('group creation failed') } - const tracker = m.getTracker(database) + const tracker = components.getTracker(database) const key = 'test' const put = database.store.creators.put(key, 'my-data') @@ -122,19 +61,19 @@ describe('groups', () => { assert.equal(await tracker.validate(key, put), true) - await sigint.interupt() + await components.stop() }) it('tracker processes a group\'s content', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) + const { components } = await create() + const group = await createGroup(components, 'test') + const database = components.groups.get(group) if (database == null) { throw new Error('group creation failed') } - const tracker = m.getTracker(database) + const tracker = components.getTracker(database) const key = '/test' const value = 'my-data' const put = database.store.creators.put(key, value) @@ -149,19 +88,19 @@ describe('groups', () => { entries = await all(tracker.process()) assert.deepEqual(entries, []) - await sigint.interupt() + await components.stop() }) it('tracker is scope limited', async () => { - const { groups: m, sigint } = await create() - const group = await mkGroup(m, 'test') - const database = m.groups.get(group) + const { components } = await create() + const group = await createGroup(components, 'test') + const database = components.groups.get(group) if (database == null) { throw new Error('group creation failed') } - const tracker = m.getTracker(database) + const tracker = components.getTracker(database) const key = '/test' const value = 'my-data' const put = database.store.creators.put(key, value) @@ -174,110 +113,110 @@ describe('groups', () => { entries = await all(tracker.process(key)) assert.deepEqual(entries, [{ key, value: cborg.encode(value) }]) - await sigint.interupt() + await components.stop() }) it('uses the identity from base in welo', async () => { - const { groups: m, sigint, base } = await create() + const { components } = await create() - assert.deepEqual(m.welo.identity, await base.keyManager.getWeloIdentity()) + assert.deepEqual(components.welo.identity, await components.keyManager.getWeloIdentity()) - await sigint.interupt() + await components.stop() }) it('rpc - id returns the base58btc formatted welo id', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const id = await client.rpc.request('id', {}) - assert.equal(uint8ArrayToString(m.welo.identity.id, 'base58btc'), id) + assert.equal(uint8ArrayToString(components.welo.identity.id, 'base58btc'), id) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - create groups creates a group without other peers', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const name = 'test' const cid = await client.rpc.request('create-group', { name, peers: [] }) const group = CID.parse(cid) - const database = m.groups.get(group) + const database = components.groups.get(group) assert(database != null) assert.equal(database.manifest.name, name) - assert.deepEqual(database.manifest.access.config?.write, [m.welo.identity.id]) + assert.deepEqual(database.manifest.access.config?.write, [components.welo.identity.id]) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - create groups creates a group with other peers', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const name = 'test' const otherPeer = 'GZsJqUjmbVqZCUMbJoe5ye4xfdKZVPVwBoFFQiyCZYesq6Us5b' const cid = await client.rpc.request('create-group', { name, peers: [otherPeer] }) const group = CID.parse(cid) - const database = m.groups.get(group) + const database = components.groups.get(group) assert(database != null) assert.equal(database.manifest.name, name) assert.deepEqual(database.manifest.access.config?.write, [ - m.welo.identity.id, + components.welo.identity.id, uint8ArrayFromString(otherPeer, 'base58btc') ]) client.close() - await sigint.interupt() + await components.stop() }) it('rpc - joins an external group', async () => { - const components = await Promise.all([create(), create('server-join-group')]) - const client = createNetClient(components[0].argv.socket) + const components = await Promise.all([create(), create()]) + const client = createNetClient(components[0].socket) const name = 'test' - const group = await mkGroup(components[1].groups, name) + const group = await createGroup(components[1].components, name) - await components[0].network.libp2p.dial(components[1].network.libp2p.getMultiaddrs()) + await components[0].components.libp2p.dial(components[1].components.libp2p.getMultiaddrs()) const res = await client.rpc.request('join-group', { group: group.toString() }) assert.equal(res, null) - const database = components[0].groups.groups.get(group) + const database = components[0].components.groups.get(group) assert(database) assert.equal(database.manifest.name, name) client.close() - await Promise.all(components.map(async c => c.sigint.interupt())) + await Promise.all(components.map(async c => c.components.stop())) }) it('rpc - list groups', async () => { - const { groups: m, sigint, argv } = await create() - const client = createNetClient(argv.socket) + const { components, socket } = await create() + const client = createNetClient(socket) const name = 'test' let groups = await client.rpc.request('list-groups', {}) assert.deepEqual(groups, []) - const group = await mkGroup(m, name) + const group = await createGroup(components, name) groups = await client.rpc.request('list-groups', {}) assert.deepEqual(groups, [{ group: group.toString(), name }]) client.close() - await sigint.interupt() + await components.stop() }) // This fails it github too - seems to think the `server-sync-groups` socket is in use? it.skip('rpc - sync groups', async () => { - const components = await Promise.all([create(), create('server-sync-groups')]) - const client = createNetClient(components[0].argv.socket) + const components = await Promise.all([create(), create()]) + const client = createNetClient(components[0].socket) const key = '/test' const value = 'test-value' @@ -285,8 +224,8 @@ describe('groups', () => { assert.deepEqual(groups, []) - const group = await mkGroup(components[1].groups, 'test') - const database = components[1].groups.groups.get(group) + const group = await createGroup(components[1].components, 'test') + const database = components[1].components.groups.get(group) if (database == null) { throw new Error('database creation failed') @@ -296,7 +235,7 @@ describe('groups', () => { await database.replica.write(put) - await components[0].network.libp2p.dial(components[1].network.libp2p.getMultiaddrs()) + await components[0].components.libp2p.dial(components[1].components.libp2p.getMultiaddrs()) await client.rpc.request('join-group', { group: group.toString() }) await client.rpc.request('sync', {}) @@ -306,6 +245,6 @@ describe('groups', () => { assert.deepEqual(result, value) client.close() - await Promise.all(components.map(async c => c.sigint.interupt())) + await Promise.all(components.map(async c => c.components.stop())) }) }) diff --git a/packages/daemon/test/modules/mock-argv.ts b/packages/daemon/test/modules/mock-argv.ts deleted file mode 100644 index c6da9904..00000000 --- a/packages/daemon/test/modules/mock-argv.ts +++ /dev/null @@ -1,12 +0,0 @@ -import Path from 'path' -import type { Provides } from '../../src/modules/argv/index.js' - -const provs = (path: string): Provides => { - return { - key: Path.join(path, 'key.json'), - config: Path.join(path, 'config.json'), - socket: Path.join(path, 'server.socket') - } -} - -export default provs diff --git a/packages/daemon/test/modules/mock-base.ts b/packages/daemon/test/modules/mock-base.ts deleted file mode 100644 index a139b417..00000000 --- a/packages/daemon/test/modules/mock-base.ts +++ /dev/null @@ -1,20 +0,0 @@ -import Path from 'path' -import { KeyManager, parseKeyData } from '@organicdesign/db-key-manager' -import { MemoryBlockstore } from 'blockstore-core' -import { FsBlockstore } from 'blockstore-fs' -import { MemoryDatastore } from 'datastore-core' -import { FsDatastore } from 'datastore-fs' -import type { Provides } from '../../src/modules/base/index.js' - -const provs = (config?: { path?: string, keyManager?: KeyManager }): Provides => { - return { - keyManager: config?.keyManager ?? new KeyManager(parseKeyData({ - key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' - })), - datastore: config?.path == null ? new MemoryDatastore() : new FsDatastore(Path.join(config?.path, 'datastore')), - blockstore: config?.path == null ? new MemoryBlockstore() : new FsBlockstore(Path.join(config?.path, 'blockstore')) - } -} - -export default provs diff --git a/packages/daemon/test/modules/mock-config.ts b/packages/daemon/test/modules/mock-config.ts deleted file mode 100644 index 45bf3666..00000000 --- a/packages/daemon/test/modules/mock-config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { type z } from 'zod' -import type { Provides } from '../../src/modules/config/index.js' - -const provs = (config: Record): Provides => ({ - config, - - get (schema: T): z.infer { - return schema.parse(config) - } -}) - -export default provs diff --git a/packages/daemon/test/modules/network.spec.ts b/packages/daemon/test/modules/network.spec.ts index 5b83c21d..330b6c57 100644 --- a/packages/daemon/test/modules/network.spec.ts +++ b/packages/daemon/test/modules/network.spec.ts @@ -1,94 +1,72 @@ import assert from 'assert/strict' import fs from 'fs/promises' +import Path from 'path' import { unixfs } from '@helia/unixfs' +import { createKeyManager } from '@organicdesign/db-key-manager' import { createNetClient } from '@organicdesign/net-rpc' import { createHelia } from 'helia' import { Key } from 'interface-datastore' import all from 'it-all' import { CID } from 'multiformats/cid' -import network from '../../src/modules/network/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' -import type { Requires as NetworkComponents } from '../../src/modules/network/index.js' import type { Libp2p } from '@libp2p/interface' -import createLibp2p from '@/modules/network/libp2p.js' +import setup from '@/common/index.js' +import { Config, type Components } from '@/common/interface.js' +import createLibp2p from '@/common/libp2p.js' describe('network', () => { const testPath = mkTestPath('network') - let components: NetworkComponents & { argv: ReturnType } + const socket = Path.join(testPath, 'server.socket') - before(async () => { + beforeEach(async () => { await fs.mkdir(testPath, { recursive: true }) - - const sigint = await createSigint() - const config = mockConfig({ storage: ':memory:' }) - const argv = mockArgv(testPath) - const base = mockBase() - const rpc = await createRpc({ argv, sigint }) - - components = { - argv, - sigint, - config, - base, - rpc - } }) - after(async () => { - await components.sigint.interupt() - await new Promise(resolve => setTimeout(resolve, 1000)) + afterEach(async () => { await fs.rm(testPath, { recursive: true }) }) it('provides defaults for the config options', async () => { - const m = await network(components) + const components = await setup({ socket }) + const config = components.parseConfig(Config) - assert.deepEqual(m.config.addresses, [ + assert.deepEqual(config.addresses, [ '/ip4/127.0.0.1/tcp/0', '/ip4/127.0.0.1/tcp/0/ws' ]) - assert.deepEqual(m.config.bootstrap, []) - assert.equal(m.config.private, false) - assert.equal(m.config.serverMode, false) + assert.deepEqual(config.bootstrap, []) + assert.equal(config.private, false) + assert.equal(config.serverMode, false) + + await components.stop() }) it('uses the stores derived form base for helia', async () => { - const m = await network(components) + const components = await setup({ socket }) const cid = CID.parse('QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ') - assert(!(await m.helia.blockstore.has(cid))) + assert(!(await components.helia.blockstore.has(cid))) + + await components.blockstore.put(cid, new Uint8Array()) - await components.base.blockstore.put(cid, new Uint8Array()) + assert(await components.helia.blockstore.has(cid)) - assert(await m.helia.blockstore.has(cid)) + assert(!(await components.helia.datastore.has(new Key('/test')))) - assert(!(await m.helia.datastore.has(new Key('/test')))) + await components.datastore.put(new Key('/helia/datastore/test'), new Uint8Array()) - await components.base.datastore.put(new Key('/helia/datastore/test'), new Uint8Array()) + assert(await components.helia.datastore.has(new Key('/test'))) - assert(await m.helia.datastore.has(new Key('/test'))) + await components.stop() }) it('is not connectable from outside when private is set to true', async () => { - const config = mockConfig({ private: true }) - - const m = await network({ - ...components, - config - }) - + const components = await setup({ socket, config: { private: true } }) const libp2p = await createLibp2p({}) - const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) - - const dialTo = libp2p.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }) - const dialFrom = m.libp2p.dial(libp2p.getMultiaddrs(), { signal: mkSignal() }) + const dialTo = libp2p.dial(components.libp2p.getMultiaddrs(), { signal: mkSignal() }) + const dialFrom = components.libp2p.dial(libp2p.getMultiaddrs(), { signal: mkSignal() }) await Promise.all([ assert.rejects(async () => dialTo), @@ -96,41 +74,32 @@ describe('network', () => { ]) await libp2p.stop() + await components.stop() }) it('is connectable from outside when private is set to false', async () => { - const config = mockConfig({ private: false }) - - const m = await network({ - ...components, - config - }) - + const components = await setup({ socket, config: { private: false } }) const [libp2p1, libp2p2] = await Promise.all([createLibp2p({}), createLibp2p({})]) const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }), - m.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) + libp2p1.dial(components.libp2p.getMultiaddrs(), { signal: mkSignal() }), + components.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) ]) await Promise.all([ libp2p1.stop(), - libp2p2.stop() + libp2p2.stop(), + components.stop() ]) }) it('is connectable from inside when private is set to true', async () => { - const config = mockConfig({ private: true }) - - const m = await network({ - ...components, - config - }) + const components = await setup({ socket, config: { private: true } }) const create = async (): Promise => createLibp2p({ - psk: components.base.keyManager.getPskKey() + psk: components.keyManager.getPskKey() }) const [libp2p1, libp2p2] = await Promise.all([create(), create()]) @@ -138,33 +107,31 @@ describe('network', () => { const mkSignal = (): AbortSignal => AbortSignal.timeout(1000) await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs(), { signal: mkSignal() }), - m.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) + libp2p1.dial(components.libp2p.getMultiaddrs(), { signal: mkSignal() }), + components.libp2p.dial(libp2p2.getMultiaddrs(), { signal: mkSignal() }) ]) await Promise.all([ libp2p1.stop(), - libp2p2.stop() + libp2p2.stop(), + components.stop() ]) }) it('bootstraps when a bootstrap peer is set', async () => { const libp2p = await createLibp2p({}) - - const config = mockConfig({ - private: false, - bootstrap: libp2p.getMultiaddrs().map(a => a.toString()) - }) - - const m = await network({ - ...components, - config + const components = await setup({ + socket, + config: { + private: false, + bootstrap: libp2p.getMultiaddrs().map(a => a.toString()) + } }) const peer = await new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 5000) - m.libp2p.addEventListener('peer:connect', (a) => { + components.libp2p.addEventListener('peer:connect', (a) => { resolve(a.detail.toBytes()) }) }) @@ -172,6 +139,7 @@ describe('network', () => { assert.deepEqual(peer, libp2p.peerId.toBytes()) await libp2p.stop() + await components.stop() }) // Something is failing inside websockets... @@ -181,17 +149,17 @@ describe('network', () => { createLibp2p({ addresses: ['/ip4/127.0.0.1/tcp/0/ws'] }) ]) - const m = await network({ - ...components, - config: mockConfig({ + const components = await setup({ + socket, + config: { private: false, serverMode: true - }) + } }) await Promise.all([ - libp2p1.dial(m.libp2p.getMultiaddrs()), - libp2p2.dial(m.libp2p.getMultiaddrs()) + libp2p1.dial(components.libp2p.getMultiaddrs()), + libp2p2.dial(components.libp2p.getMultiaddrs()) ]) await new Promise(resolve => setTimeout(resolve, 5000)) @@ -200,64 +168,63 @@ describe('network', () => { await Promise.all([ libp2p1.stop(), - libp2p2.stop() + libp2p2.stop(), + components.stop() ]) }) it('libp2p remembers peers with persistant storage', async () => { const libp2p = await createLibp2p({}) - const sigints = await Promise.all([createSigint(), createSigint()]) + const keyManager = await createKeyManager() - const create = async (index: number): ReturnType => network({ - ...components, - - sigint: sigints[index], - - base: mockBase({ path: testPath }), - - config: mockConfig({ - private: false - }) + const start = async (): Promise => setup({ + socket, + keyManager, + config: { + private: false, + storage: testPath + } }) - const m1 = await create(0) + let components = await start() - await libp2p.dial(m1.libp2p.getMultiaddrs()) + await libp2p.dial(components.libp2p.getMultiaddrs()) - const [peer] = m1.libp2p.getPeers() + const [peer] = components.libp2p.getPeers() - await m1.libp2p.peerStore.save(peer, peer) + await components.libp2p.peerStore.save(peer, peer) - await sigints[0].interupt() + await components.stop() - const m2 = await create(1) + components = await start() - const peers = await m2.libp2p.peerStore.all() + const [saved] = await components.libp2p.peerStore.all() - assert.deepEqual(peers[0].id.toBytes(), peer.toBytes()) + assert.deepEqual(saved.id.toBytes(), peer.toBytes()) - await sigints[1].interupt() + await components.stop() await libp2p.stop() }) it('rpc - addresses returns the peers addresses', async () => { - const m = await network(components) - const client = createNetClient(components.argv.socket) + const components = await setup({ socket, config: { private: false } }) + const client = createNetClient(socket) const addresses = await client.rpc.request('addresses', {}) - assert.deepEqual(addresses, m.libp2p.getMultiaddrs().map(a => a.toString())) + assert.deepEqual(addresses, components.libp2p.getMultiaddrs().map(a => a.toString())) client.close() + await components.stop() }) it('rpc - connections returns the peers connections', async () => { const libp2p = await createLibp2p({}) - const m = await network(components) - const client = createNetClient(components.argv.socket) + const components = await setup({ socket, config: { private: false } }) + const client = createNetClient(socket) assert.deepEqual(await client.rpc.request('connections', {}), []) - await libp2p.dial(m.libp2p.getMultiaddrs()) + await libp2p.dial(components.libp2p.getMultiaddrs()) const connections = await client.rpc.request('connections', {}) @@ -265,26 +232,28 @@ describe('network', () => { assert.deepEqual( connections, - m.libp2p.getConnections().map(a => a.remoteAddr.toString()) + components.libp2p.getConnections().map(a => a.remoteAddr.toString()) ) await libp2p.stop() client.close() + await components.stop() }) it('rpc - connection connects to another peer', async () => { const libp2p = await createLibp2p({}) - const m = await network(components) - const client = createNetClient(components.argv.socket) + const components = await setup({ socket, config: { private: false } }) + const client = createNetClient(socket) await client.rpc.request('connect', { address: libp2p.getMultiaddrs()[0] }) - const connections = m.libp2p.getConnections() + const connections = components.libp2p.getConnections() assert.equal(connections.length, 1) assert.deepEqual(connections[0].remotePeer.toBytes(), libp2p.peerId.toBytes()) await libp2p.stop() + await components.stop() client.close() }) @@ -294,10 +263,10 @@ describe('network', () => { const libp2p = await createLibp2p({}) const helia = await createHelia({ libp2p }) const ufs = unixfs(helia) - const m = await network(components) - const client = createNetClient(components.argv.socket) + const components = await setup({ socket, config: { private: false } }) + const client = createNetClient(socket) - await m.libp2p.dial(helia.libp2p.getMultiaddrs()) + await components.libp2p.dial(helia.libp2p.getMultiaddrs()) const cid = await ufs.addBytes(data) await all(helia.pins.add(cid)) @@ -308,6 +277,7 @@ describe('network', () => { await helia.stop() await libp2p.stop() + await components.stop() client.close() }) }) diff --git a/packages/daemon/test/modules/revisions.spec.ts b/packages/daemon/test/modules/revisions.spec.ts index 3392b4bb..7c08f5e4 100644 --- a/packages/daemon/test/modules/revisions.spec.ts +++ b/packages/daemon/test/modules/revisions.spec.ts @@ -2,7 +2,6 @@ import assert from 'assert' import fs from 'fs/promises' import Path from 'path' import { unixfs } from '@helia/unixfs' -import { KeyManager } from '@organicdesign/db-key-manager' import * as testData from '@organicdesign/db-test-utils' import { importer } from '@organicdesign/db-utils' import { createNetClient } from '@organicdesign/net-rpc' @@ -10,101 +9,31 @@ import all from 'it-all' import { CID } from 'multiformats/cid' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import createDownloader from '../../src/modules/downloader/index.js' -import createFilesystem from '../../src/modules/filesystem/index.js' -import createGroups from '../../src/modules/groups/index.js' -import createNetwork from '../../src/modules/network/index.js' -import createRevisions from '../../src/modules/revisions/index.js' -import createRpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' -import createTick from '../../src/modules/tick/index.js' import { createGroup } from '../utils/create-group.js' import { createDag } from '../utils/dag.js' -import { generateKey } from '../utils/generate-key.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import mockBase from './mock-base.js' -import mockConfig from './mock-config.js' +import type { Components } from '@/common/interface.js' +import type { Context as FilesystemContext } from '@/modules/filesystem/index.js' +import type { Context as RevisionsContext } from '@/modules/revisions/index.js' +import setup from '@/common/index.js' +import setupFilesystem from '@/modules/filesystem/index.js' +import setupRevisions from '@/modules/revisions/index.js' describe('revisions', () => { const testPath = mkTestPath('revisions') - const create = async (name?: string): Promise<{ - argv: ReturnType - config: ReturnType - rpc: Awaited> - base: ReturnType - network: Awaited> - groups: Awaited> - filesystem: Awaited> - sigint: Awaited> - tick: Awaited> - revisions: Awaited> + const create = async (): Promise<{ + revisions: RevisionsContext + filesystem: FilesystemContext + components: Components + socket: string }> => { - const path = name == null ? testPath : Path.join(testPath, name) + const socket = Path.join(testPath, `${Math.random()}.socket`) + const components = await setup({ socket }) + const filesystem = await setupFilesystem(components) + const revisions = await setupRevisions(components) - const keyManager = new KeyManager(await generateKey()) - - await fs.mkdir(path, { recursive: true }) - - const argv = mockArgv(path) - const config = mockConfig({ storage: ':memory:' }) - const sigint = await createSigint() - const rpc = await createRpc({ argv, sigint }) - const base = mockBase({ keyManager }) - const network = await createNetwork({ config, sigint, base, rpc }) - - const groups = await createGroups({ - sigint, - base, - rpc, - network - }) - - const downloader = await createDownloader({ - sigint, - base, - rpc, - network, - config - }) - - const tick = await createTick({ config, sigint }) - - const filesystem = await createFilesystem({ - sigint, - base, - rpc, - network, - groups, - downloader, - tick, - config - }) - - const revisions = await createRevisions({ - base, - network, - rpc, - groups, - filesystem, - config, - downloader, - tick - }) - - return { - argv, - config, - sigint, - rpc, - base, - network, - groups, - filesystem, - tick, - revisions - } + return { filesystem, revisions, components, socket } } before(async () => { @@ -116,30 +45,30 @@ describe('revisions', () => { }) it('returns null when wraping a group that doesn\'t exist', async () => { - const { revisions: m, sigint } = await create() - const r = m.getRevisions(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) + const { revisions, components } = await create() + const r = revisions.getRevisions(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN')) assert.equal(r, null) - await sigint.interupt() + await components.stop() }) it('wraps a group in revisions', async () => { - const { revisions: m, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const r = m.getRevisions(group) + const { revisions, components } = await create() + const group = await createGroup(components, 'test') + const r = revisions.getRevisions(group) assert.notEqual(r, null) - await sigint.interupt() + await components.stop() }) it('creates a revision when a file is added to the filesystem', async () => { - const { revisions: m, filesystem, network, groups, sigint } = await create() - const group = await createGroup(groups, 'test') - const r = m.getRevisions(group) + const { revisions, filesystem, components } = await create() + const group = await createGroup(components, 'test') + const r = revisions.getRevisions(group) const fs = filesystem.getFileSystem(group) - const dag = await createDag(network.helia, 2, 2) + const dag = await createDag(components.helia, 2, 2) const path = '/test' assert(r != null) @@ -148,7 +77,7 @@ describe('revisions', () => { const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, path, { @@ -171,7 +100,7 @@ describe('revisions', () => { assert.deepEqual(entries, [{ path, sequence: 0, - author: groups.welo.identity.id, + author: components.welo.identity.id, entry: { cid: entry.cid, @@ -183,27 +112,27 @@ describe('revisions', () => { } }]) - await sigint.interupt() + await components.stop() }) it('rpc - it exports a revision (file)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') + const { filesystem, components, socket } = await create() + const group = await createGroup(components, 'test') const fs = filesystem.getFileSystem(group) const path = '/test' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const sequence = 0 const dataFile = testData.data[0] const exportPath = dataFile.generatePath(testPath) assert(fs != null) - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + const [{ cid }] = await all(importer(components.helia.blockstore, dataFile.path)) const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, path, { @@ -220,7 +149,7 @@ describe('revisions', () => { const response = await client.rpc.request('export-revision', { group: group.toString(), path, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc'), + author: uint8ArrayToString(components.welo.identity.id, 'base58btc'), sequence, outPath: exportPath }) @@ -231,15 +160,15 @@ describe('revisions', () => { assert.equal(valid, true) - await sigint.interupt() + await components.stop() }) it('rpc - it exports a revision (directory)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') + const { filesystem, components, socket } = await create() + const group = await createGroup(components, 'test') const fs = filesystem.getFileSystem(group) const rootPath = '/test' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const sequence = 0 const outPath = Path.join(testPath, 'export-directory') @@ -248,12 +177,12 @@ describe('revisions', () => { for (const dataFile of testData.data) { const virtualPath = dataFile.generatePath(rootPath) - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + const [{ cid }] = await all(importer(components.helia.blockstore, dataFile.path)) const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, virtualPath, { @@ -271,7 +200,7 @@ describe('revisions', () => { const response = await client.rpc.request('export-revision', { group: group.toString(), path: rootPath, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc'), + author: uint8ArrayToString(components.welo.identity.id, 'base58btc'), sequence, outPath }) @@ -285,25 +214,25 @@ describe('revisions', () => { assert.equal(valid, true) } - await sigint.interupt() + await components.stop() }) it('rpc - lists a revision (file)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') + const { filesystem, components, socket } = await create() + const group = await createGroup(components, 'test') const fs = filesystem.getFileSystem(group) const path = '/test' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const dataFile = testData.data[0] assert(fs != null) - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + const [{ cid }] = await all(importer(components.helia.blockstore, dataFile.path)) const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) const before = Date.now() @@ -326,7 +255,7 @@ describe('revisions', () => { assert(Array.isArray(response)) assert.equal(response.length, 1) - assert.equal(response[0].author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(response[0].author, uint8ArrayToString(components.welo.identity.id, 'base58btc')) assert.equal(response[0].blocks, 1) assert.equal(response[0].cid, cid.toString()) assert.equal(response[0].encrypted, false) @@ -337,15 +266,15 @@ describe('revisions', () => { assert(response[0].timestamp >= before) assert(response[0].timestamp <= Date.now()) - await sigint.interupt() + await components.stop() }) it('rpc - it lists a revision (directory)', async () => { - const { network, filesystem, groups, sigint, argv } = await create() - const group = await createGroup(groups, 'test') + const { filesystem, components, socket } = await create() + const group = await createGroup(components, 'test') const fs = filesystem.getFileSystem(group) const rootPath = '/test' - const client = createNetClient(argv.socket) + const client = createNetClient(socket) assert(fs != null) @@ -354,12 +283,12 @@ describe('revisions', () => { for (const dataFile of testData.data) { const virtualPath = dataFile.generatePath(rootPath) - const [{ cid }] = await all(importer(network.helia.blockstore, dataFile.path)) + const [{ cid }] = await all(importer(components.helia.blockstore, dataFile.path)) const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, virtualPath, { @@ -383,7 +312,7 @@ describe('revisions', () => { assert.equal(response.length, 3) for (const item of response) { - assert.equal(item.author, uint8ArrayToString(groups.welo.identity.id, 'base58btc')) + assert.equal(item.author, uint8ArrayToString(components.welo.identity.id, 'base58btc')) assert.equal(item.blocks, 1) assert.equal(item.encrypted, false) assert.equal(item.priority, 1) @@ -401,14 +330,14 @@ describe('revisions', () => { assert(BigInt(item.size) === dataFile.size) } - await sigint.interupt() + await components.stop() }) it('rpc - read revision', async () => { - const { filesystem, groups, network, sigint, argv } = await create() - const client = createNetClient(argv.socket) - const group = await createGroup(groups, 'test') - const ufs = unixfs(network.helia) + const { filesystem, components, socket } = await create() + const client = createNetClient(socket) + const group = await createGroup(components, 'test') + const ufs = unixfs(components.helia) const path = '/test' const data = 'test-data' @@ -417,7 +346,7 @@ describe('revisions', () => { const promise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error('timeout')) }, 100) - filesystem.events.addEventListener('file:added', () => { resolve() }, { once: true }) + components.events.addEventListener('file:added', () => { resolve() }, { once: true }) }) await filesystem.uploads.add('put', [group.bytes, path, { @@ -435,7 +364,7 @@ describe('revisions', () => { group: group.toString(), path, sequence: 0, - author: uint8ArrayToString(groups.welo.identity.id, 'base58btc') + author: uint8ArrayToString(components.welo.identity.id, 'base58btc') } const read1 = await client.rpc.request('read', coreParams) @@ -455,6 +384,6 @@ describe('revisions', () => { assert.deepEqual(read4, data.slice(1, 3 + 1)) client.close() - await sigint.interupt() + await components.stop() }) }) diff --git a/packages/daemon/test/modules/rpc.spec.ts b/packages/daemon/test/modules/rpc.spec.ts index da8324cc..2b707095 100644 --- a/packages/daemon/test/modules/rpc.spec.ts +++ b/packages/daemon/test/modules/rpc.spec.ts @@ -1,51 +1,34 @@ import assert from 'assert/strict' import fs from 'fs/promises' +import Path from 'path' import { createNetClient } from '@organicdesign/net-rpc' -import rpc from '../../src/modules/rpc/index.js' -import createSigint from '../../src/modules/sigint/index.js' import { mkTestPath } from '../utils/paths.js' -import mockArgv from './mock-argv.js' -import type { Provides as Argv } from '../../src/modules/argv/index.js' -import type { Provides as Sigint } from '../../src/modules/sigint/index.js' +import setup from '@/common/index.js' const testPath = mkTestPath('rpc') describe('rpc', () => { - let argv: Argv - let sigint: Sigint + const socket = Path.join(testPath, 'server.socket') before(async () => { - argv = mockArgv(testPath) - sigint = await createSigint() - await fs.mkdir(testPath, { recursive: true }) - - await fs.writeFile(argv.key, JSON.stringify({ - key: '5TP9VimJU1WdSoTxZGLhSuPKqCpXirPHDK4ZjHxzetex-8zAV14C4oLe4dytUSVzznTuQ659pY1dSMG8HAQenDqVQ', - psk: '/key/swarm/psk/1.0.0/\n/base16/\n56d3c18282f1f1b1b3e04e40dd5d8bf44cafa8bc9c9bc7c57716a7766fa2c550' - })) }) after(async () => { await fs.rm(testPath, { recursive: true }) - await sigint.interupt() }) it('adds RPC methods', async () => { + const components = await setup({ socket }) const testData = { key: 'value' } const returnData = { return: 'return-value' } - const m = await rpc({ - argv, - sigint - }) - - const client = createNetClient(argv.socket) + const client = createNetClient(socket) const methodPromise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error()) }, 50) - m.addMethod('test', async params => { + components.net.rpc.addMethod('test', async params => { resolve(params) return returnData }) @@ -57,5 +40,6 @@ describe('rpc', () => { assert.deepEqual(await methodPromise, testData) client.close() + await components.stop() }) }) diff --git a/packages/daemon/test/modules/sigint.spec.ts b/packages/daemon/test/modules/sigint.spec.ts deleted file mode 100644 index 2aa2c68a..00000000 --- a/packages/daemon/test/modules/sigint.spec.ts +++ /dev/null @@ -1,15 +0,0 @@ -import assert from 'assert/strict' -import sigint from '../../src/modules/sigint/index.js' - -describe('sigint', () => { - it('calls the interupt method when interupt gets called', async () => { - const m = await sigint() - - let called = 0 - - m.onInterupt(() => called++) - await m.interupt() - - assert.equal(called, 1) - }) -}) diff --git a/packages/daemon/test/modules/tick.spec.ts b/packages/daemon/test/modules/tick.spec.ts index bf22fe15..02b1fa04 100644 --- a/packages/daemon/test/modules/tick.spec.ts +++ b/packages/daemon/test/modules/tick.spec.ts @@ -1,46 +1,11 @@ import assert from 'assert/strict' -import { type z } from 'zod' -import setupSigint from '../../src/modules/sigint/index.js' -import tick from '../../src/modules/tick/index.js' -import mockConfig from './mock-config.js' +import { createTick } from '../../src/common/tick.js' describe('tick', () => { - it('returns default tick interval', async () => { - const sigint = await setupSigint() - - const m = await tick({ - config: { config: {}, get: (schema: z.AnyZodObject) => schema.parse({}) }, - sigint - }) - - assert.deepEqual(m.config, { tickInterval: 600 }) - - await sigint.interupt() - }) - - it('returns config tick interval', async () => { - const tickInterval = 100 - const sigint = await setupSigint() - - const m = await tick({ - config: mockConfig({ tickInterval }), - sigint - }) - - assert.deepEqual(m.config, { tickInterval }) - - await sigint.interupt() - }) - it('returns ticks every interval', async () => { const tickInterval = 5 const checkTimes = 6 - const sigint = await setupSigint() - - const m = await tick({ - config: mockConfig({ tickInterval }), - sigint - }) + const tick = await createTick(tickInterval) const before = Date.now() @@ -48,7 +13,7 @@ describe('tick', () => { setTimeout(() => { reject(new Error('timeout')) }, (checkTimes + 4) * tickInterval) let timesCalled = 0 - m.register(() => { + tick.add(() => { timesCalled++ if (timesCalled >= checkTimes) { @@ -59,11 +24,11 @@ describe('tick', () => { const after = Date.now() + await tick.stop() + const delta = after - before assert(delta < tickInterval * (checkTimes + 2)) assert(delta > tickInterval * (checkTimes - 2)) - - await sigint.interupt() }) }) diff --git a/packages/daemon/test/utils/create-group.ts b/packages/daemon/test/utils/create-group.ts index b708735d..4aa4d24a 100644 --- a/packages/daemon/test/utils/create-group.ts +++ b/packages/daemon/test/utils/create-group.ts @@ -1,17 +1,17 @@ import { type CID } from 'multiformats/cid' -import type { Provides as GroupsProvides } from '../../src/modules/groups/index.js' +import type { Components } from '../../src/common/interface.js' -export const createGroup = async (m: GroupsProvides, name: string, peers: Uint8Array[] = []): Promise => { - const manifest = await m.welo.determine({ +export const createGroup = async ({ welo, groups }: Components, name: string, peers: Uint8Array[] = []): Promise => { + const manifest = await welo.determine({ name, meta: { type: 'group' }, access: { protocol: '/hldb/access/static', - config: { write: [m.welo.identity.id, ...peers] } + config: { write: [welo.identity.id, ...peers] } } }) - await m.groups.add(manifest) + await groups.add(manifest) return manifest.address.cid } diff --git a/packages/key-manager/src/key-manager.ts b/packages/key-manager/src/key-manager.ts index 4bf1b8ae..97d5387b 100644 --- a/packages/key-manager/src/key-manager.ts +++ b/packages/key-manager/src/key-manager.ts @@ -3,7 +3,13 @@ import { keys } from '@libp2p/crypto' import { encode as encodeBlock } from 'multiformats/block' import { sha256 } from 'multiformats/hashes/sha2' import { Identity } from 'welo' -import { importKeyFile, keyToPeerId } from './utils.js' +import { + importKeyFile, + keyToPeerId, + generateKeyData, + generateMnemonic, + parseKeyData +} from './utils.js' import type { KeyData } from './interface.js' import type { PeerId } from '@libp2p/interface' import type { BIP32Interface } from 'bip32' @@ -90,7 +96,13 @@ export class KeyManager { } } -export const createKeyManager = async (path: string): Promise => { +export const createKeyManager = async (path?: string): Promise => { + if (path == null) { + const keys = await generateKeyData(generateMnemonic(), generateMnemonic()[0]) + + return new KeyManager(parseKeyData(keys)) + } + const keys = await importKeyFile(path) return new KeyManager(keys) diff --git a/packages/utils/src/importer.ts b/packages/utils/src/importer.ts index 7fb03edf..4dd10774 100644 --- a/packages/utils/src/importer.ts +++ b/packages/utils/src/importer.ts @@ -8,9 +8,7 @@ import type { ImportResult } from 'ipfs-unixfs-importer' export const importer = async function * (blockstore: Blockstore, path: string, options: AddOptions = {}): AsyncIterable { const ufs = unixfs({ blockstore }) - - const stat = await fs.lstat(path) - + const stat = await fs.stat(path) const globPattern = stat.isFile() ? path.split('/').pop() ?? '' : '**/*' if (stat.isFile()) { @@ -21,7 +19,7 @@ export const importer = async function * (blockstore: Blockstore, path: string, assert(src.path != null) const rPath = Path.join(path, src.path) - const stat = await fs.lstat(rPath) + const stat = await fs.stat(rPath) if (stat.isDirectory()) { continue