From b5f1f8d0511b038400bfc4f19385ad7f72d1d93a Mon Sep 17 00:00:00 2001 From: ScottyPoi Date: Thu, 5 Dec 2024 15:04:54 -0700 Subject: [PATCH 1/3] remove testdouble dependency --- packages/portalnetwork/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/portalnetwork/package.json b/packages/portalnetwork/package.json index f73daeaff..31412c39c 100644 --- a/packages/portalnetwork/package.json +++ b/packages/portalnetwork/package.json @@ -69,7 +69,6 @@ "c8": "^7.12.0", "eslint": "^8.6.0", "js-yaml": "^4.1.0", - "testdouble": "^3.16.3", "tslib": "^2.3.1", "typedoc": "^0.24.0", "typedoc-plugin-markdown": "^3.11.3", From 6a0c504bebefad378a262b6e8c9105b946862a31 Mon Sep 17 00:00:00 2001 From: ScottyPoi Date: Thu, 5 Dec 2024 15:05:06 -0700 Subject: [PATCH 2/3] delete testdouble based tests --- .../networks/history/historyNetwork.spec.ts | 58 +--- .../test/networks/protocol.spec.ts | 308 ------------------ 2 files changed, 1 insertion(+), 365 deletions(-) delete mode 100644 packages/portalnetwork/test/networks/protocol.spec.ts diff --git a/packages/portalnetwork/test/networks/history/historyNetwork.spec.ts b/packages/portalnetwork/test/networks/history/historyNetwork.spec.ts index 01ced5fe8..1f0e02cdc 100644 --- a/packages/portalnetwork/test/networks/history/historyNetwork.spec.ts +++ b/packages/portalnetwork/test/networks/history/historyNetwork.spec.ts @@ -1,16 +1,12 @@ import { readFileSync } from 'fs' import { createRequire } from 'module' -import { EntryStatus } from '@chainsafe/discv5' -import { ENR } from '@chainsafe/enr' import { Block, type BlockBytes, BlockHeader } from '@ethereumjs/block' import * as RLP from '@ethereumjs/rlp' -import { bytesToHex, concatBytes, hexToBytes } from '@ethereumjs/util' -import * as td from 'testdouble' +import { bytesToHex, hexToBytes } from '@ethereumjs/util' import { assert, describe, it } from 'vitest' import { BlockHeaderWithProof, - ContentKeyType, HistoricalRootsBlockProof, HistoryNetworkContentType, NetworkId, @@ -28,58 +24,6 @@ import type { HistoryNetwork } from '../../../src/index.js' const require = createRequire(import.meta.url) const testBlocks = require('../../testData/testBlocksForHistory.json') -describe('history Network FINDCONTENT/FOUNDCONTENT message handlers', async () => { - const block1Rlp = testBlocks.block1.blockRlp - const block1Hash = testBlocks.block1.blockHash - const node = await PortalNetwork.create({ - bindAddress: '192.168.0.1', - transport: TransportLayer.WEB, - supportedNetworks: [{ networkId: NetworkId.HistoryNetwork }], - }) - - const network = node.networks.get(NetworkId.HistoryNetwork) as HistoryNetwork - const remoteEnr = - 'enr:-IS4QG_M1lzTXzQQhUcAViqK-WQKtBgES3IEdQIBbH6tlx3Zb-jCFfS1p_c8Xq0Iie_xT9cHluSyZl0TNCWGlUlRyWcFgmlkgnY0gmlwhKRc9EGJc2VjcDI1NmsxoQMo1NBoJfVY367ZHKA-UBgOE--U7sffGf5NBsNSVG629oN1ZHCCF6Q' - const decodedEnr = ENR.decodeTxt(remoteEnr) - network.routingTable.insertOrUpdate(decodedEnr, EntryStatus.Connected) - const key = getContentKey( - HistoryNetworkContentType.BlockBody, - hexToBytes('0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6'), - ) - - const findContentResponse = Uint8Array.from([5, 1, 97, 98, 99]) - network.store = td.func() - network.validateHeader = td.func() - // network.sendFindContent = td.func() - network.sendMessage = td.func() - - td.when( - network.sendMessage(td.matchers.anything(), td.matchers.anything(), td.matchers.anything()), - ).thenResolve(findContentResponse) - const res = await network.sendFindContent(decodedEnr.nodeId, key) - it('should send a FINDCONTENT message', () => { - assert.exists(res!['content']) - assert.deepEqual( - res!['content'], - Uint8Array.from([97, 98, 99]), - 'got correct response for content abc', - ) - }) - - // TODO: Write good `handleFindContent` tests - - td.reset() - const headerKey = getContentKey(HistoryNetworkContentType.BlockHeader, hexToBytes(block1Hash)) - await network.store(headerKey, hexToBytes(block1Rlp)) - const contentKey = ContentKeyType.serialize( - concatBytes(Uint8Array.from([HistoryNetworkContentType.BlockHeader]), hexToBytes(block1Hash)), - ) - const header = await network.sendFindContent('0xabcd', contentKey) - it('should send a FINDCONTENT message for a block header', () => { - assert.equal(header, undefined, 'received undefined for unknown peer') - }) -}) - describe('store -- Headers and Epoch Accumulators', async () => { it('Should store and retrieve block header from DB', async () => { // const epochKey = '0x035ec1ffb8c3b146f42606c74ced973dc16ec5a107c0345858c343fc94780b4218' diff --git a/packages/portalnetwork/test/networks/protocol.spec.ts b/packages/portalnetwork/test/networks/protocol.spec.ts deleted file mode 100644 index 68973664b..000000000 --- a/packages/portalnetwork/test/networks/protocol.spec.ts +++ /dev/null @@ -1,308 +0,0 @@ -import { EntryStatus } from '@chainsafe/discv5' -import { ENR, SignableENR } from '@chainsafe/enr' -import { hexToBytes } from '@ethereumjs/util' -import { keys } from '@libp2p/crypto' -import { multiaddr } from '@multiformats/multiaddr' -import { MemoryLevel } from 'memory-level' -import * as td from 'testdouble' -import { assert, describe, it } from 'vitest' - -import { - ContentRequest, - MessageCodes, - NetworkId, - PingPongCustomDataType, - PortalNetwork, - PortalWireMessageType, - TransportLayer, - generateRandomNodeIdAtDistance, -} from '../../src/index.js' - -import type { BitArray } from '@chainsafe/ssz' -import type { AbstractLevel } from 'abstract-level' -import type { BaseNetwork, HistoryNetwork, NodesMessage } from '../../src/index.js' - -describe('network wire message tests', async () => { - const node = await PortalNetwork.create({ - bindAddress: '192.168.0.1', - transport: TransportLayer.WEB, - supportedNetworks: [{ networkId: NetworkId.HistoryNetwork }], - }) - const baseNetwork = node.networks.get(NetworkId.HistoryNetwork) as BaseNetwork - it('BaseNetwork', async () => { - baseNetwork.sendMessage = td.func() - td.when( - baseNetwork.sendMessage( - td.matchers.anything(), - td.matchers.anything(), - td.matchers.anything(), - ), - ).thenResolve(hexToBytes('0x1234')) - const res = await baseNetwork.sendMessage('enr', new Uint8Array(), NetworkId.HistoryNetwork) - assert.deepEqual(res, hexToBytes('0x1234'), 'sendMessage should return the response') - }) - - it('PING/PONG message handlers', async () => { - const network = baseNetwork as any - const remoteEnr = - 'enr:-IS4QG_M1lzTXzQQhUcAViqK-WQKtBgES3IEdQIBbH6tlx3Zb-jCFfS1p_c8Xq0Iie_xT9cHluSyZl0TNCWGlUlRyWcFgmlkgnY0gmlwhKRc9EGJc2VjcDI1NmsxoQMo1NBoJfVY367ZHKA-UBgOE--U7sffGf5NBsNSVG629oN1ZHCCF6Q' - const decodedEnr = ENR.decodeTxt(remoteEnr) - const pongResponse = Uint8Array.from([ - 1, 5, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]) - network.sendMessage = td.func() - td.when( - network.sendMessage(td.matchers.anything(), td.matchers.anything(), td.matchers.anything()), - ).thenResolve(pongResponse) - const res = await network.sendPing(decodedEnr) - assert.equal(res?.enrSeq, 5n, 'received expected PONG response') - assert.equal(res?.customPayload[0], 1, 'received expected PONG response') - const payload = PingPongCustomDataType.serialize({ radius: BigInt(1) }) - const msg = { - selector: MessageCodes.PING, - value: { - enrSeq: node.discv5.enr.seq, - customPayload: payload, - }, - } - const nodeAddr = { - socketAddr: decodedEnr.getLocationMultiaddr('udp'), - nodeId: decodedEnr.nodeId, - } - network.sendPong = td.func() - td.when(network.sendPong(nodeAddr, 1n)).thenDo(() => - assert.ok(true, 'correctly handled PING message'), - ) - network.updateRoutingTable = td.func() - network.handlePing(nodeAddr, 1n, msg.value) - }) - - it('FINDNODES/NODES message handlers', async () => { - const network = baseNetwork as any - const remoteEnr = - 'enr:-IS4QG_M1lzTXzQQhUcAViqK-WQKtBgES3IEdQIBbH6tlx3Zb-jCFfS1p_c8Xq0Iie_xT9cHluSyZl0TNCWGlUlRyWcFgmlkgnY0gmlwhKRc9EGJc2VjcDI1NmsxoQMo1NBoJfVY367ZHKA-UBgOE--U7sffGf5NBsNSVG629oN1ZHCCF6Q' - const decodedEnr = ENR.decodeTxt(remoteEnr) - network.routingTable.insertOrUpdate(decodedEnr, EntryStatus.Connected) - const findNodesResponse = Uint8Array.from([ - 3, 1, 5, 0, 0, 0, 4, 0, 0, 0, 248, 132, 184, 64, 98, 28, 68, 73, 123, 43, 66, 88, 148, 220, - 175, 197, 99, 155, 158, 245, 113, 112, 19, 145, 242, 62, 9, 177, 46, 127, 179, 172, 15, 214, - 73, 120, 117, 10, 84, 236, 35, 36, 1, 7, 157, 133, 186, 53, 153, 250, 87, 144, 208, 228, 233, - 233, 190, 215, 71, 114, 119, 169, 10, 2, 182, 117, 100, 246, 5, 130, 105, 100, 130, 118, 52, - 130, 105, 112, 132, 127, 0, 0, 1, 137, 115, 101, 99, 112, 50, 53, 54, 107, 49, 161, 2, 166, - 64, 119, 30, 57, 36, 215, 222, 189, 27, 126, 14, 93, 46, 164, 80, 142, 10, 84, 179, 46, 141, - 1, 3, 181, 22, 178, 254, 0, 158, 156, 232, 131, 117, 100, 112, 130, 158, 250, - ]) - network.sendMessage = td.func() - td.when( - network.sendMessage(td.matchers.anything(), td.matchers.anything(), td.matchers.anything()), - ).thenResolve(findNodesResponse) - let res = await network.sendFindNodes(decodedEnr.nodeId, [0, 1, 2]) - assert.equal(res.total, 1, 'received 1 ENR from FINDNODES') - res = await network.sendFindNodes( - 'c875efa288b97fce46c93adbeb05b25465acfe00121ec00f6db7f3bd883ac6f2', - [], - ) - assert.equal(res, undefined, 'received undefined when no valid NODES response received') - - network.sendResponse = td.func() - const findNodesMessageWithDistance = { distances: [2, 4, 0, 0, 0, 0, 0] } - const findNodesMessageWithoutDistance = { distances: [2, 4, 0, 0, 0] } - node.discv5.enr.encode = td.func() - td.when( - network.sendResponse( - { socketAddr: multiaddr(), nodeId: 'abc' }, - td.matchers.anything(), - td.matchers.argThat((arg: Uint8Array) => arg.length > 3), - ), - ).thenDo(() => assert.ok(true, 'correctly handle findNodes message with ENRs')) - td.when( - network.sendResponse( - { socketAddr: multiaddr(), nodeId: 'abc' }, - td.matchers.anything(), - td.matchers.argThat((arg: Uint8Array) => arg.length === 0), - ), - ).thenDo(() => assert.ok(true, 'correctly handle findNodes message with no ENRs')) - td.when(node.discv5.enr.encode()).thenReturn(Uint8Array.from([0, 1, 2])) - network.handleFindNodes( - { socketAddr: multiaddr(), nodeId: 'abc' }, - 1n, - findNodesMessageWithDistance, - ) - network.handleFindNodes( - { socketAddr: multiaddr(), nodeId: 'abc' }, - 1n, - findNodesMessageWithoutDistance, - ) - }) - - td.reset() - - it('OFFER/ACCEPT message handlers', async () => { - const network = baseNetwork as any - let res = await network.sendOffer( - 'c875efa288b97fce46c93adbeb05b25465acfe00121ec00f6db7f3bd883ac6f2', - [], - ) - assert.equal(res, undefined, 'received undefined when no invalid ENR provided') - - const remoteEnr = - 'enr:-IS4QG_M1lzTXzQQhUcAViqK-WQKtBgES3IEdQIBbH6tlx3Zb-jCFfS1p_c8Xq0Iie_xT9cHluSyZl0TNCWGlUlRyWcFgmlkgnY0gmlwhKRc9EGJc2VjcDI1NmsxoQMo1NBoJfVY367ZHKA-UBgOE--U7sffGf5NBsNSVG629oN1ZHCCF6Q' - const decodedEnr = ENR.decodeTxt(remoteEnr) - network.routingTable.insertOrUpdate(decodedEnr, EntryStatus.Connected) - network.sendMessage = td.func() - const acceptResponse = Uint8Array.from([7, 229, 229, 6, 0, 0, 0, 3]) - td.when( - network.sendMessage(td.matchers.anything(), td.matchers.anything(), td.matchers.anything()), - ).thenResolve(acceptResponse) - - network.handleNewRequest = td.func() - td.when( - network.handleNewRequest({ - contentKeys: td.matchers.anything(), - peerId: td.matchers.contains('abc'), - connectionId: td.matchers.anything(), - contents: td.matchers.anything(), - }), - ).thenResolve(ContentRequest.prototype) - res = await network.sendOffer(decodedEnr.nodeId, [Uint8Array.from([1])]) - assert.deepEqual( - (res as BitArray).uint8Array, - Uint8Array.from([1]), - 'received valid ACCEPT response to OFFER', - ) - - const noWantResponse = Uint8Array.from([7, 229, 229, 6, 0, 0, 0, 0]) - td.when( - network.sendMessage(td.matchers.anything(), td.matchers.anything(), td.matchers.anything()), - ).thenResolve(noWantResponse) - res = await network.sendOffer(decodedEnr.nodeId, [Uint8Array.from([0])]) - assert.equal(res, undefined, 'received undefined when no valid ACCEPT message received') - }) -}) -describe('handleFindNodes message handler tests', async () => { - const node = await PortalNetwork.create({ - bindAddress: '192.168.0.1', - transport: TransportLayer.WEB, - supportedNetworks: [{ networkId: NetworkId.HistoryNetwork }], - }) - const network = node.networks.get(NetworkId.HistoryNetwork) as HistoryNetwork - type sendResponse = (src: any, requestId: bigint, payload: Uint8Array) => Promise - network.sendResponse = td.func() - - const sortedEnrs: ENR[] = [] - - it('test FindNodes', async () => { - for (let x = 239; x < 257; x++) { - const id = generateRandomNodeIdAtDistance(node.discv5.enr.nodeId, x) - const privKey = await keys.generateKeyPair('secp256k1') - const enr = SignableENR.createFromPrivateKey(privKey) - const remoteEnr = enr.toENR() - ;(remoteEnr as any).nodeId = id - sortedEnrs.push(remoteEnr) - - network.routingTable.insertOrUpdate(remoteEnr, EntryStatus.Connected) - } - const newNode = generateRandomNodeIdAtDistance(node.discv5.enr.nodeId, 0) - await (network as any).handleFindNodes({ socketAddr: multiaddr(), nodeId: newNode }, 1n, { - distances: [239], - }) - td.verify( - network.sendResponse( - { socketAddr: multiaddr(), nodeId: newNode }, - 1n, - td.matchers.argThat((arg: Uint8Array) => { - const msg = PortalWireMessageType.deserialize(arg).value as NodesMessage - return msg.enrs.length === 1 - }), - ), - ) - assert.ok( - true, - 'Nodes response contained no ENRs since should be nothing in table at distance 239', - ) - - td.reset() - - network.sendResponse = td.func() - await (network as any).handleFindNodes({ socketAddr: multiaddr(), nodeId: newNode }, 1n, { - distances: [255, 256], - }) - - td.verify( - network.sendResponse( - { socketAddr: multiaddr(), nodeId: newNode }, - 1n, - td.matchers.argThat((arg: Uint8Array) => { - const msg = PortalWireMessageType.deserialize(arg).value as NodesMessage - return msg.enrs.length === 2 - }), - ), - ) - assert.ok(true, 'Nodes response contained 2 ENRs since should be one node in each bucket') - td.reset() - - const id = generateRandomNodeIdAtDistance(node.discv5.enr.nodeId, 255) - const privKey = await keys.generateKeyPair('secp256k1') - const enr = SignableENR.createFromPrivateKey(privKey) - enr.encode() - ;(enr as any)._nodeId = id - network.routingTable.insertOrUpdate(enr.toENR(), EntryStatus.Connected) - - await (network as any).handleFindNodes({ socketAddr: multiaddr(), nodeId: newNode }, 1n, { - distances: [255, 256], - }) - - td.verify( - network.sendResponse( - { socketAddr: multiaddr(), nodeId: newNode }, - 1n, - td.matchers.argThat((arg: Uint8Array) => { - const msg = PortalWireMessageType.deserialize(arg).value as NodesMessage - return msg.enrs.length > 0 - }), - ), - ) - assert.ok(true, 'Nodes response contained 3 ENRs since one more ENR added to bucket 256') - - await (network as any).handleFindNodes({ socketAddr: multiaddr(), nodeId: newNode }, 1n, { - distances: [239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 255, 256], - }) - - td.verify( - network.sendResponse( - { socketAddr: multiaddr(), nodeId: newNode }, - 1n, - td.matchers.argThat((arg: Uint8Array) => { - const msg = PortalWireMessageType.deserialize(arg).value as NodesMessage - return msg.enrs.length === 10 - }), - ), - ) - assert.ok( - true, - 'Nodes response contained 10 ENRs even though requested nodes in 12 buckets since nodes max payload size met', - ) - }) -}) - -describe('stored radius', async () => { - const r = 2n ** 254n - 1n - const db = new MemoryLevel() - await db.put('radius', r.toString()) - const node = await PortalNetwork.create({ - bindAddress: '192.168.0.1', - transport: TransportLayer.WEB, - supportedNetworks: [ - { - networkId: NetworkId.HistoryNetwork, - db: { db: db as AbstractLevel, path: '' }, - }, - ], - }) - const history = node.networks.get(NetworkId.HistoryNetwork) as HistoryNetwork - const radius = history.nodeRadius - it('should have correct radius', () => { - assert.equal(radius, r, 'correct radius') - }) -}) From 5329aaf26b53e96691acca2547375dfc6aba7fc4 Mon Sep 17 00:00:00 2001 From: ScottyPoi Date: Thu, 5 Dec 2024 15:26:10 -0700 Subject: [PATCH 3/3] DB: ignore non-hex keys during size() --- package-lock.json | 65 ------------------- .../portalnetwork/src/networks/networkDB.ts | 8 ++- 2 files changed, 6 insertions(+), 67 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2be885174..2b2b1d9d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6796,14 +6796,6 @@ "node": ">=8" } }, - "node_modules/is-plain-obj": { - "version": "1.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-regex": { "version": "1.1.4", "license": "MIT", @@ -6818,14 +6810,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-regexp": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-shared-array-buffer": { "version": "1.0.3", "license": "MIT", @@ -7215,11 +7199,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash": { - "version": "4.17.21", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.merge": { "version": "4.6.2", "dev": true, @@ -8434,18 +8413,6 @@ ], "license": "MIT" }, - "node_modules/quibble": { - "version": "0.9.2", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash": "^4.17.21", - "resolve": "^1.22.8" - }, - "engines": { - "node": ">= 0.14.0" - } - }, "node_modules/react-is": { "version": "18.3.1", "dev": true, @@ -9081,18 +9048,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/stringify-object-es5": { - "version": "2.5.0", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "is-plain-obj": "^1.0.0", - "is-regexp": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/strip-ansi": { "version": "6.0.1", "license": "MIT", @@ -9250,30 +9205,11 @@ "node": ">=8" } }, - "node_modules/testdouble": { - "version": "3.20.2", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash": "^4.17.21", - "quibble": "^0.9.2", - "stringify-object-es5": "^2.5.0", - "theredoc": "^1.0.0" - }, - "engines": { - "node": ">= 16" - } - }, "node_modules/text-table": { "version": "0.2.0", "dev": true, "license": "MIT" }, - "node_modules/theredoc": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, "node_modules/through": { "version": "2.3.8", "license": "MIT" @@ -10340,7 +10276,6 @@ "c8": "^7.12.0", "eslint": "^8.6.0", "js-yaml": "^4.1.0", - "testdouble": "^3.16.3", "tslib": "^2.3.1", "typedoc": "^0.24.0", "typedoc-plugin-markdown": "^3.11.3", diff --git a/packages/portalnetwork/src/networks/networkDB.ts b/packages/portalnetwork/src/networks/networkDB.ts index e5fb662cc..77657a6fb 100644 --- a/packages/portalnetwork/src/networks/networkDB.ts +++ b/packages/portalnetwork/src/networks/networkDB.ts @@ -133,8 +133,12 @@ export class NetworkDB { const _db = this.db as MemoryLevel let size = 0 for await (const [key, value] of _db.iterator()) { - size += hexToBytes('0x' + padToEven(key.slice(2))).length - size += hexToBytes(value).length + try { + size += hexToBytes('0x' + padToEven(key.slice(2))).length + size += hexToBytes(value).length + } catch { + // ignore + } } return size }