feat: add random walk component (#2501)

To allow finding network services, add a random walk component that
lets services find random network peers in a scalable way.

If two services try to random walk at the same time, they will share
the results.

packages/interface-internal/src/index.ts

@@ -1,5 +1,6 @@
 export * from './address-manager/index.js'
 export * from './connection-manager/index.js'
+export * from './random-walk/index.js'
 export * from './record/index.js'
 export * from './registrar/index.js'
 export * from './transport-manager/index.js'

packages/interface-internal/src/random-walk/index.ts

@@ -0,0 +1,13 @@
+import type { AbortOptions, PeerInfo } from '@libp2p/interface'
+
+/**
+ * RandomWalk finds random peers on the network and dials them. Use this after
+ * registering a Topology if you need to discover common network services.
+ */
+export interface RandomWalk {
+  /**
+   * Begin or join an existing walk. Abort the passed signal if you wish to
+   * abort the walk early.
+   */
+  walk(options?: AbortOptions): AsyncGenerator<PeerInfo>
+}

packages/libp2p/package.json

@@ -105,6 +105,9 @@
     "it-parallel": "^3.0.6",
     "merge-options": "^3.0.4",
     "multiformats": "^13.1.0",
+    "p-defer": "^4.0.1",
+    "race-event": "^1.3.0",
+    "race-signal": "^1.0.2",
     "uint8arrays": "^5.0.3"
   },
   "devDependencies": {
@@ -121,11 +124,11 @@
     "delay": "^6.0.0",
     "it-all": "^3.0.4",
     "it-drain": "^3.0.5",
-    "it-map": "^3.0.5",
+    "it-map": "^3.1.0",
     "it-pipe": "^3.0.1",
     "it-pushable": "^3.2.3",
     "it-stream-types": "^2.0.1",
-    "p-defer": "^4.0.1",
+    "it-take": "^3.0.4",
     "p-event": "^6.0.1",
     "p-wait-for": "^5.0.2",
     "sinon": "^17.0.1",

packages/libp2p/src/libp2p.ts

@@ -17,6 +17,7 @@ import { DefaultConnectionManager } from './connection-manager/index.js'
 import { CompoundContentRouting } from './content-routing.js'
 import { codes } from './errors.js'
 import { DefaultPeerRouting } from './peer-routing.js'
+import { RandomWalk } from './random-walk.js'
 import { DefaultRegistrar } from './registrar.js'
 import { DefaultTransportManager } from './transport-manager.js'
 import { DefaultUpgrader } from './upgrader.js'
@@ -137,6 +138,9 @@ export class Libp2pNode<T extends ServiceMap = Record<string, unknown>> extends
       routers: contentRouters
     }))
 
+    // Random walk
+    this.configureComponent('randomWalk', new RandomWalk(this.components))
+
     // Discovery modules
     ;(init.peerDiscovery ?? []).forEach((fn, index) => {
       const service = this.configureComponent(`peer-discovery-${index}`, fn(this.components))

packages/libp2p/src/random-walk.ts

@@ -0,0 +1,138 @@
+import { randomBytes } from '@libp2p/crypto'
+import { TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
+import { anySignal } from 'any-signal'
+import pDefer, { type DeferredPromise } from 'p-defer'
+import { raceEvent } from 'race-event'
+import { raceSignal } from 'race-signal'
+import type { AbortOptions, ComponentLogger, Logger, PeerInfo, PeerRouting, Startable } from '@libp2p/interface'
+import type { RandomWalk as RandomWalkInterface } from '@libp2p/interface-internal'
+
+export interface RandomWalkComponents {
+  peerRouting: PeerRouting
+  logger: ComponentLogger
+}
+
+interface RandomWalkEvents {
+  'walk:peer': CustomEvent<PeerInfo>
+  'walk:error': CustomEvent<Error>
+}
+
+export class RandomWalk extends TypedEventEmitter<RandomWalkEvents> implements RandomWalkInterface, Startable {
+  private readonly peerRouting: PeerRouting
+  private readonly log: Logger
+  private walking: boolean
+  private walkers: number
+  private shutdownController: AbortController
+  private walkController?: AbortController
+  private needNext?: DeferredPromise<void>
+
+  constructor (components: RandomWalkComponents) {
+    super()
+
+    this.log = components.logger.forComponent('libp2p:random-walk')
+    this.peerRouting = components.peerRouting
+    this.walkers = 0
+    this.walking = false
+
+    // stops any in-progress walks when the node is shut down
+    this.shutdownController = new AbortController()
+    setMaxListeners(Infinity, this.shutdownController.signal)
+  }
+
+  start (): void {
+    this.shutdownController = new AbortController()
+    setMaxListeners(Infinity, this.shutdownController.signal)
+  }
+
+  stop (): void {
+    this.shutdownController.abort()
+  }
+
+  async * walk (options?: AbortOptions): AsyncGenerator<PeerInfo> {
+    if (!this.walking) {
+      // start the query that causes walk:peer events to be emitted
+      this.startWalk()
+    }
+
+    this.walkers++
+    const signal = anySignal([this.shutdownController.signal, options?.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      while (true) {
+        // if another consumer has paused the query, start it again
+        this.needNext?.resolve()
+        this.needNext = pDefer()
+
+        // wait for a walk:peer or walk:error event
+        const event = await raceEvent<CustomEvent<PeerInfo>>(this, 'walk:peer', signal, {
+          errorEvent: 'walk:error'
+        })
+
+        yield event.detail
+      }
+    } finally {
+      signal.clear()
+      this.walkers--
+
+      // stop the walk if no more consumers are interested
+      if (this.walkers === 0) {
+        this.walkController?.abort()
+        this.walkController = undefined
+      }
+    }
+  }
+
+  private startWalk (): void {
+    this.walking = true
+
+    // the signal for this controller will be aborted if no more random peers
+    // are required
+    this.walkController = new AbortController()
+    setMaxListeners(Infinity, this.walkController.signal)
+
+    const signal = anySignal([this.walkController.signal, this.shutdownController.signal])
+    setMaxListeners(Infinity, signal)
+
+    const start = Date.now()
+    let found = 0
+
+    Promise.resolve().then(async () => {
+      this.log('start walk')
+
+      // find peers until no more consumers are interested
+      while (this.walkers > 0) {
+        try {
+          for await (const peer of this.peerRouting.getClosestPeers(randomBytes(32), { signal })) {
+            signal.throwIfAborted()
+
+            this.log('found peer %p', peer.id)
+            found++
+            this.safeDispatchEvent('walk:peer', {
+              detail: peer
+            })
+
+            // if we only have one consumer, pause the query until they request
+            // another random peer or they signal they are no longer interested
+            if (this.walkers === 1 && this.needNext != null) {
+              await raceSignal(this.needNext.promise, signal)
+            }
+          }
+        } catch (err) {
+          this.log.error('randomwalk errored', err)
+
+          this.safeDispatchEvent('walk:error', {
+            detail: err
+          })
+        }
+      }
+    })
+      .catch(err => {
+        this.log.error('randomwalk errored', err)
+      })
+      .finally(() => {
+        this.log('finished walk, found %d peers after %dms', found, Date.now() - start)
+        this.walking = false
+      })
+  }
+}