From 0d8469427af33a052db68a2bca3494b46ca77dc3 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Thu, 20 Jul 2023 23:32:44 +0530 Subject: [PATCH 1/5] feat: Headers-Sync --- bin/bcoin | 5 +- bin/neutrino | 43 +++++ lib/bcoin-browser.js | 1 + lib/bcoin.js | 1 + lib/blockchain/chain.js | 37 ++++- lib/blockchain/chaindb.js | 63 +++++++- lib/blockchain/layout.js | 3 + lib/net/peer.js | 12 ++ lib/net/pool.js | 111 +++++++++++-- lib/node/neutrino.js | 319 ++++++++++++++++++++++++++++++++++++++ lib/protocol/networks.js | 2 +- 11 files changed, 577 insertions(+), 20 deletions(-) create mode 100755 bin/neutrino create mode 100644 lib/node/neutrino.js diff --git a/bin/bcoin b/bin/bcoin index 0df74c54f..990d72f19 100755 --- a/bin/bcoin +++ b/bin/bcoin @@ -43,7 +43,10 @@ for arg in "$@"; do --daemon) daemon=1 ;; - --spv) + --neutrino) + cmd='neutrino' + ;; + --spv) cmd='spvnode' ;; esac diff --git a/bin/neutrino b/bin/neutrino new file mode 100755 index 000000000..7872de324 --- /dev/null +++ b/bin/neutrino @@ -0,0 +1,43 @@ +#!/usr/bin/env node + +'use strict'; + +console.log('Starting bcoin'); +process.title = 'bcoin'; +const Neutrino = require('../lib/node/neutrino'); + +const node = new Neutrino({ + file: true, + argv: true, + env: true, + logFile: true, + logConsole: true, // todo: remove + logLevel: 'debug', // todo: remove + db: 'leveldb', + memory: false, + workers: true, + loader: require +}); + +if (!node.config.bool('no-wallet') && !node.has('walletdb')) { + const plugin = require('../lib/wallet/plugin'); + node.use(plugin); +} + +(async () => { + await node.ensure(); + await node.open(); + await node.connect(); + node.startSync(); +})().catch((err) => { + console.error(err.stack); + process.exit(1); +}); + +process.on('unhandledRejection', (err, promise) => { + throw err; +}); + +process.on('SIGINT', async () => { + await node.close(); +}); diff --git a/lib/bcoin-browser.js b/lib/bcoin-browser.js index 1f7254be8..8b2d46cb5 100644 --- a/lib/bcoin-browser.js +++ b/lib/bcoin-browser.js @@ -89,6 +89,7 @@ bcoin.node = require('./node'); bcoin.Node = require('./node/node'); bcoin.FullNode = require('./node/fullnode'); bcoin.SPVNode = require('./node/spvnode'); +bcoin.Neutrino = require('./node/neutrino'); // Primitives bcoin.primitives = require('./primitives'); diff --git a/lib/bcoin.js b/lib/bcoin.js index a2edf9e78..4cc519e74 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -123,6 +123,7 @@ bcoin.define('node', './node'); bcoin.define('Node', './node/node'); bcoin.define('FullNode', './node/fullnode'); bcoin.define('SPVNode', './node/spvnode'); +bcoin.define('Neutrino', './node/neutrino'); // Primitives bcoin.define('primitives', './primitives'); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 9cd0a312f..6537e72ef 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -1791,6 +1791,24 @@ class Chain extends AsyncEmitter { return this.hasEntry(hash); } + async getCFHeaderHeight() { + return await this.db.getCFHeaderHeight(); + } + + async saveCFHeaderHeight(height) { + this.db.neutrinoState.headerHeight = height; + await this.db.saveNeutrinoState(); + } + + async getCFilterHeight() { + return await this.db.getCFilterHeight(); + } + + async saveCFilterHeight(height) { + this.db.neutrinoState.filterHeight = height; + await this.db.saveNeutrinoState(); + } + /** * Find the corresponding block entry by hash or height. * @param {Hash|Number} hash/height @@ -2003,19 +2021,22 @@ class Chain extends AsyncEmitter { if (this.synced) return; + if (this.options.neutrino && this.getProgress() < 1) + return; if (this.options.checkpoints) { if (this.height < this.network.lastCheckpoint) return; - } - - if (this.tip.time < util.now() - this.network.block.maxTipAge) + } else if (!this.options.neutrino && + this.tip.time < util.now() - this.network.block.maxTipAge) return; if (!this.hasChainwork()) return; - this.synced = true; - this.emit('full'); + if (this.options.neutrino) + this.emit('headersFull'); + else + this.emit('full'); } /** @@ -2616,6 +2637,7 @@ class ChainOptions { this.compression = true; this.spv = false; + this.neutrino = false; this.bip91 = false; this.bip148 = false; this.prune = false; @@ -2662,6 +2684,11 @@ class ChainOptions { this.spv = options.spv; } + if (options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.prefix != null) { assert(typeof options.prefix === 'string'); this.prefix = options.prefix; diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index cb91accaa..c4d2e0671 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -46,6 +46,7 @@ class ChainDB { this.state = new ChainState(); this.pending = null; this.current = null; + this.neutrinoState = null; this.cacheHash = new LRU(this.options.entryCache, null, BufferMap); this.cacheHeight = new LRU(this.options.entryCache); @@ -90,6 +91,11 @@ class ChainDB { this.logger.info('ChainDB successfully initialized.'); } + if (this.options.neutrino) { + if (!this.neutrinoState) + this.neutrinoState = await this.getNeutrinoState(); + } + this.logger.info( 'Chain State: hash=%h tx=%d coin=%d value=%s.', this.state.tip, @@ -1001,7 +1007,7 @@ class ChainDB { */ async getRawBlock(block) { - if (this.options.spv) + if (this.options.spv && !this.options.neutrino) return null; const hash = await this.getHash(block); @@ -1670,6 +1676,39 @@ class ChainDB { b.put(layout.O.encode(), flags.toRaw()); return b.write(); } + + /** + * Get Neutrino State + * @returns {Promise} - Returns neutrino state + */ + + async getNeutrinoState() { + const data = await this.db.get(layout.N.encode()); + if (!data) + return new NeutrinoState(); + return NeutrinoState.fromRaw(data); + } + + async getCFHeaderHeight() { + const state = await this.getNeutrinoState(); + return state.headerHeight; + } + + async getCFilterHeight() { + const state = await this.getNeutrinoState(); + return state.filterHeight; + } + + /** + * Save Neutrino State + * @returns {void} + */ + async saveNeutrinoState() { + const state = this.neutrinoState.toRaw(); + const b = this.db.batch(); + b.put(layout.N.encode(), state); + return b.write(); + } } /** @@ -1952,6 +1991,28 @@ function fromU32(num) { return data; } +class NeutrinoState { + constructor() { // TODO: do we add support for multiple filters? + this.headerHeight = 0; + this.filterHeight = 0; + } + + toRaw() { + const bw = bio.write(8); + bw.writeU32(this.headerHeight); + bw.writeU32(this.filterHeight); + return bw.render(); + } + + static fromRaw(data) { + const state = new NeutrinoState(); + const br = bio.read(data); + state.headerHeight = br.readU32(); + state.filterHeight = br.readU32(); + return state; + } +} + /* * Expose */ diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index 337f95900..532ccb050 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -14,6 +14,7 @@ const bdb = require('bdb'); * O -> chain options * R -> tip hash * D -> versionbits deployments + * N -> neutrino state * e[hash] -> entry * h[hash] -> height * H[height] -> hash @@ -33,6 +34,8 @@ const layout = { O: bdb.key('O'), R: bdb.key('R'), D: bdb.key('D'), + N: bdb.key('N'), + F: bdb.key('H', ['hash256']), e: bdb.key('e', ['hash256']), h: bdb.key('h', ['hash256']), H: bdb.key('H', ['uint32']), diff --git a/lib/net/peer.js b/lib/net/peer.js index 2271e7896..dac2e265d 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1449,6 +1449,12 @@ class Peer extends EventEmitter { if (!(this.services & services.NETWORK)) throw new Error('Peer does not support network services.'); + if (this.options.neutrino) { + if (!(this.services & services.NODE_COMPACT_FILTERS)) { + throw new Error('Peer does not support Compact Filters.'); + } + } + if (this.options.headers) { if (this.version < common.HEADERS_VERSION) throw new Error('Peer does not support getheaders.'); @@ -2080,6 +2086,7 @@ class PeerOptions { this.agent = common.USER_AGENT; this.noRelay = false; this.spv = false; + this.neutrino = false; this.compact = false; this.headers = false; this.banScore = common.BAN_SCORE; @@ -2143,6 +2150,11 @@ class PeerOptions { this.spv = options.spv; } + if (options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.compact != null) { assert(typeof options.compact === 'boolean'); this.compact = options.compact; diff --git a/lib/net/pool.js b/lib/net/pool.js index 234b23bc2..fcf11a3b8 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -204,16 +204,22 @@ class Pool extends EventEmitter { */ resetChain() { - if (!this.options.checkpoints) + if (!this.options.checkpoints && !this.options.neutrino) return; - this.checkpoints = false; + if (!this.options.neutrino) + this.checkpoints = false; this.headerTip = null; this.headerChain.reset(); this.headerNext = null; const tip = this.chain.tip; - + if (this.options.neutrino) { + this.headerChain.push(new HeaderEntry(tip.hash, tip.height)); + this.cfHeaderChain = new List(); + this.cfHeaderChain.push(new CFHeaderEntry(consensus.ZERO_HASH, 0)); + return; + } if (tip.height < this.network.lastCheckpoint) { this.checkpoints = true; this.headerTip = this.getNextTip(tip.height); @@ -650,7 +656,10 @@ class Pool extends EventEmitter { return; this.syncing = true; - this.resync(false); + if (this.options.neutrino) { + this.startHeadersSync(); + } else + this.resync(false); } /** @@ -704,6 +713,32 @@ class Pool extends EventEmitter { this.compactBlocks.clear(); } + /** + * Start the headers sync using getHeaders messages. + * @private + * @return {Promise} + */ + + async startHeadersSync() { + if (!this.syncing) + return; + let locator; + try { + locator = await this.chain.getLocator(); + } catch (e) { + this.emit('error', e); + return; + } + + const peer = this.peers.load; + if (!peer) { + this.logger.info('No loader peer.'); + return; + } + this.chain.synced = false; + peer.sendGetHeaders(locator); + } + /** * Send a sync to each peer. * @private @@ -814,7 +849,12 @@ class Pool extends EventEmitter { peer.syncing = true; peer.blockTime = Date.now(); - + if (this.options.neutrino) { + peer.sendGetHeaders(locator); + if (!this.syncing) + this.startFilterHeadersSync(); + return true; + } if (this.checkpoints) { peer.sendGetHeaders(locator, this.headerTip.hash); return true; @@ -1634,6 +1674,13 @@ class Pool extends EventEmitter { if (this.options.hasWitness() && !peer.hasWitness()) return; + if (this.options.neutrino) { + const filterHeight = await this.chain.getCFilterHeight(); + if (filterHeight === this.chain.height) + this.startSync(); + return; + } + // Request headers instead. if (this.checkpoints) return; @@ -1698,6 +1745,8 @@ class Pool extends EventEmitter { */ async handleTXInv(peer, hashes) { + if (this.options.neutrino) + return; assert(hashes.length > 0); if (this.syncing && !this.chain.synced) @@ -2027,6 +2076,9 @@ class Pool extends EventEmitter { if (this.options.selfish) return; + if (this.options.neutrino) + return; + if (this.chain.options.spv) return; @@ -2139,7 +2191,8 @@ class Pool extends EventEmitter { async _handleHeaders(peer, packet) { const headers = packet.items; - if (!this.checkpoints) + if (!this.checkpoints && !this.options.neutrino) + // todo add support for checkpoints return; if (!this.syncing) @@ -2179,13 +2232,14 @@ class Pool extends EventEmitter { this.logger.warning( 'Peer sent a bad header chain (%s).', peer.hostname()); - peer.destroy(); + peer.increaseBan(10); return; } node = new HeaderEntry(hash, height); - if (node.height === this.headerTip.height) { + if (!this.options.neutrino && node.height === this.headerTip.height) { + // todo add support for checkpoints if (!node.hash.equals(this.headerTip.hash)) { this.logger.warning( 'Peer sent an invalid checkpoint (%s).', @@ -2200,6 +2254,8 @@ class Pool extends EventEmitter { this.headerNext = node; this.headerChain.push(node); + if (this.options.neutrino) + await this._addBlock(peer, header, chainCommon.flags.VERIFY_POW); } this.logger.debug( @@ -2212,14 +2268,19 @@ class Pool extends EventEmitter { peer.blockTime = Date.now(); // Request the blocks we just added. - if (checkpoint) { + if (checkpoint && !this.options.neutrino) { this.headerChain.shift(); this.resolveHeaders(peer); return; } // Request more headers. - peer.sendGetHeaders([node.hash], this.headerTip.hash); + if (this.chain.synced) + return; + if (this.options.neutrino) + peer.sendGetHeaders([node.hash]); + else + peer.sendGetHeaders([node.hash], this.headerTip.hash); } /** @@ -2293,7 +2354,7 @@ class Pool extends EventEmitter { const hash = block.hash(); - if (!this.resolveBlock(peer, hash)) { + if (!this.options.neutrino && !this.resolveBlock(peer, hash)) { this.logger.warning( 'Received unrequested block: %h (%s).', block.hash(), peer.hostname()); @@ -3690,6 +3751,7 @@ class PoolOptions { this.prefix = null; this.checkpoints = true; this.spv = false; + this.neutrino = false; this.bip37 = false; this.bip157 = false; this.listen = false; @@ -3772,12 +3834,17 @@ class PoolOptions { if (options.spv != null) { assert(typeof options.spv === 'boolean'); - assert(options.spv === this.chain.options.spv); this.spv = options.spv; } else { this.spv = this.chain.options.spv; } + if (options.neutrino != null) { + assert(options.compact !== true); + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.bip37 != null) { assert(typeof options.bip37 === 'boolean'); this.bip37 = options.bip37; @@ -3953,6 +4020,12 @@ class PoolOptions { this.listen = false; } + if (this.neutrino) { + this.requiredServices |= common.services.NODE_COMPACT_FILTERS; + this.checkpoints = true; + this.compact = false; + } + if (this.selfish) { this.services &= ~common.services.NETWORK; this.bip37 = false; @@ -4494,6 +4567,20 @@ class HeaderEntry { } } +class CFHeaderEntry { + /** + * Create cfheader entry. + * @constructor + */ + + constructor(hash, height) { + this.hash = hash; + this.height = height; + this.prev = null; + this.next = null; + } +} + /* * Expose */ diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js new file mode 100644 index 000000000..8077943f2 --- /dev/null +++ b/lib/node/neutrino.js @@ -0,0 +1,319 @@ +/*! + * neutrino.js - spv node for bcoin + * Copyright (c) 2023, Manav Desai (MIT License) + * Copyright (c) 2023, Shaswat Gupta (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const assert = require('bsert'); +const Chain = require('../blockchain/chain'); +const Pool = require('../net/pool'); +const Node = require('./node'); +const HTTP = require('./http'); +const RPC = require('./rpc'); +const blockstore = require('../blockstore'); +const FilterIndexer = require('../indexer/filterindexer'); + +/** + * Neutrino Node + * Create a neutrino node which only maintains + * a chain, a pool, and an http server. + * @alias module:node.Neutrino + * @extends Node + */ + +class Neutrino extends Node { + /** + * Create Neutrino node. + * @constructor + * @param {Object?} options + * @param {Buffer?} options.sslKey + * @param {Buffer?} options.sslCert + * @param {Number?} options.httpPort + * @param {String?} options.httpHost + */ + + constructor(options) { + super('bcoin', 'bcoin.conf', 'debug.log', options); + + this.opened = false; + + // SPV flag. + this.spv = false; + this.neutrino = true; + + // Instantiate block storage. + this.blocks = blockstore.create({ + network: this.network, + logger: this.logger, + prefix: this.config.prefix, + cacheSize: this.config.mb('block-cache-size'), + memory: this.memory, + spv: this.spv, + neutrino: this.neutrino + }); + + this.chain = new Chain({ + blocks: this.blocks, + network: this.network, + logger: this.logger, + prefix: this.config.prefix, + memory: this.memory, + maxFiles: this.config.uint('max-files'), + cacheSize: this.config.mb('cache-size'), + entryCache: this.config.uint('entry-cache'), + forceFlags: this.config.bool('force-flags'), + checkpoints: this.config.bool('checkpoints'), + bip91: this.config.bool('bip91'), + bip148: this.config.bool('bip148'), + spv: true, + neutrino: this.neutrino + }); + + this.filterIndexers.set( + 'BASIC', + new FilterIndexer({ + network: this.network, + logger: this.logger, + blocks: this.blocks, + chain: this.chain, + memory: this.config.bool('memory'), + prefix: this.config.str('index-prefix', this.config.prefix), + filterType: 'BASIC', + neutrino: true + }) + ); + + this.pool = new Pool({ + network: this.network, + logger: this.logger, + chain: this.chain, + prefix: this.config.prefix, + checkpoints: true, + filterIndexers: this.filterIndexers, + proxy: this.config.str('proxy'), + onion: this.config.bool('onion'), + upnp: this.config.bool('upnp'), + seeds: this.config.array('seeds'), + nodes: this.config.array('nodes'), + only: this.config.array('only'), + maxOutbound: this.config.uint('max-outbound'), + createSocket: this.config.func('create-socket'), + memory: this.memory, + selfish: true, + listen: false, + neutrino: this.neutrino, + spv: this.spv + }); + + this.rpc = new RPC(this); + + this.http = new HTTP({ + network: this.network, + logger: this.logger, + node: this, + prefix: this.config.prefix, + ssl: this.config.bool('ssl'), + keyFile: this.config.path('ssl-key'), + certFile: this.config.path('ssl-cert'), + host: this.config.str('http-host'), + port: this.config.uint('http-port'), + apiKey: this.config.str('api-key'), + noAuth: this.config.bool('no-auth'), + cors: this.config.bool('cors') + }); + + this.init(); + } + + /** + * Initialize the node. + * @private + */ + + init() { + // Bind to errors + this.chain.on('error', err => this.error(err)); + this.pool.on('error', err => this.error(err)); + + if (this.http) + this.http.on('error', err => this.error(err)); + + this.chain.on('block', (block) => { + this.emit('block', block); + }); + + this.chain.on('connect', async (entry, block) => { + this.emit('connect', entry, block); + }); + + this.chain.on('disconnect', (entry, block) => { + this.emit('disconnect', entry, block); + }); + + this.chain.on('reorganize', (tip, competitor) => { + this.emit('reorganize', tip, competitor); + }); + + this.chain.on('reset', (tip) => { + this.emit('reset', tip); + }); + + this.chain.on('headersFull', async () => { + if (this.chain.height === 0) + return; + this.logger.info('Block Headers are fully synced'); + await this.pool.startFilterHeadersSync(); + }); + + this.pool.on('cfheaders', async () => { + if (this.chain.height === 0) + return; + this.logger.info('Filter Headers are fully synced'); + await this.pool.startFilterSync(); + }); + + this.loadPlugins(); + } + + /** + * Open the node and all its child objects, + * wait for the database to load. + * @returns {Promise} + */ + + async open() { + assert(!this.opened, 'Neutrino Node is already open.'); + this.opened = true; + + await this.handlePreopen(); + await this.blocks.open(); + await this.chain.open(); + await this.pool.open(); + + await this.openPlugins(); + + await this.http.open(); + await this.handleOpen(); + + for (const filterindex of this.filterIndexers.values()) { + await filterindex.open(); + } + + this.logger.info('Node is loaded.'); + } + + /** + * Close the node, wait for the database to close. + * @returns {Promise} + */ + + async close() { + assert(this.opened, 'Neutrino Node is not open.'); + this.opened = false; + + await this.handlePreclose(); + await this.http.close(); + + await this.closePlugins(); + + await this.pool.close(); + await this.chain.close(); + await this.handleClose(); + + for (const filterindex of this.filterIndexers.values()) { + await filterindex.close(); + } + } + + /** + * Scan for any missed transactions. + * Note that this will replay the blockchain sync. + * @param {Number|Hash} start - Start block. + * @returns {Promise} + */ + + async scan(start) { + throw new Error('Not implemented.'); + } + + /** + * Broadcast a transaction (note that this will _not_ be verified + * by the mempool - use with care, lest you get banned from + * bitcoind nodes). + * @param {TX|Block} item + * @returns {Promise} + */ + + async broadcast(item) { + try { + await this.pool.broadcast(item); + } catch (e) { + this.emit('error', e); + } + } + + /** + * Broadcast a transaction (note that this will _not_ be verified + * by the mempool - use with care, lest you get banned from + * bitcoind nodes). + * @param {TX} tx + * @returns {Promise} + */ + + sendTX(tx) { + return this.broadcast(tx); + } + + /** + * Broadcast a transaction. Silence errors. + * @param {TX} tx + * @returns {Promise} + */ + + relay(tx) { + return this.broadcast(tx); + } + + /** + * Connect to the network. + * @returns {Promise} + */ + + connect() { + return this.pool.connect(); + } + + /** + * Disconnect from the network. + * @returns {Promise} + */ + + disconnect() { + return this.pool.disconnect(); + } + + /** + * Start the blockchain sync. + */ + + startSync() { + return this.pool.startSync(); + } + + /** + * Stop syncing the blockchain. + */ + + stopSync() { + return this.pool.stopSync(); + } +} + +/* + * Expose + */ + +module.exports = Neutrino; diff --git a/lib/protocol/networks.js b/lib/protocol/networks.js index 16e6bedf7..8c2db9e8e 100644 --- a/lib/protocol/networks.js +++ b/lib/protocol/networks.js @@ -792,7 +792,7 @@ regtest.block = { bip66hash: null, pruneAfterHeight: 1000, keepBlocks: 10000, - maxTipAge: 0xffffffff, + maxTipAge: 24 * 60 * 60, slowHeight: 0 }; From ed0ce73363d19128b2752c455aa970de76838c50 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Thu, 20 Jul 2023 23:41:22 +0530 Subject: [PATCH 2/5] feat: Filters-Sync --- bin/bcoin-cli | 20 ++++ lib/client/node.js | 15 ++- lib/indexer/filterindexer.js | 43 +++++++ lib/indexer/indexer.js | 14 +++ lib/net/peer.js | 66 +++++++++++ lib/net/pool.js | 222 ++++++++++++++++++++++++++++++++++- lib/node/fullnode.js | 22 ---- lib/node/http.js | 21 +++- lib/node/node.js | 44 +++++++ lib/node/rpc.js | 45 +++++++ lib/wallet/nodeclient.js | 14 +++ 11 files changed, 497 insertions(+), 29 deletions(-) diff --git a/bin/bcoin-cli b/bin/bcoin-cli index b9136ab43..9ca39cdfe 100755 --- a/bin/bcoin-cli +++ b/bin/bcoin-cli @@ -129,6 +129,22 @@ class CLI { this.log(filter); } + async getFilterHeader() { + let hash = this.config.str(0, ''); + + if (hash.length !== 64) + hash = parseInt(hash, 10); + + const filterHeader = await this.client.getFilterHeader(hash); + + if (!filterHeader) { + this.log('Filter header not found.'); + return; + } + + this.log(filterHeader); + } + async estimateFee() { const blocks = this.config.uint(0, 1); @@ -246,6 +262,9 @@ class CLI { case 'filter': await this.getFilter(); break; + case 'filterheader': + await this.getFilterHeader(); + break; case 'fee': await this.estimateFee(); break; @@ -263,6 +282,7 @@ class CLI { this.log(' $ coin [hash+index/address]: View coins.'); this.log(' $ fee [target]: Estimate smart fee.'); this.log(' $ filter [hash/height]: View filter.'); + this.log(' $ filterheader [hash/height]: View filter header.'); this.log(' $ header [hash/height]: View block header.'); this.log(' $ info: Get server info.'); this.log(' $ mempool: Get mempool snapshot.'); diff --git a/lib/client/node.js b/lib/client/node.js index 50800cac1..5df661dfa 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -164,9 +164,18 @@ class NodeClient extends Client { * @returns {Promise} */ - getFilter(filter) { - assert(typeof filter === 'string' || typeof filter === 'number'); - return this.get(`/filter/${filter}`); + getFilter(block) { + assert(typeof block === 'string' || typeof block === 'number'); + return this.get(`/filter/${block}`); + } + + getFilterHeader(block) { + assert(typeof block === 'string' || typeof block === 'number'); + return this.get(`/filterheader/${block}`); + } + + getBlockPeer(hash) { + return this.call('get block peer', hash); } /** diff --git a/lib/indexer/filterindexer.js b/lib/indexer/filterindexer.js index 97265253b..ae88af139 100644 --- a/lib/indexer/filterindexer.js +++ b/lib/indexer/filterindexer.js @@ -85,6 +85,49 @@ class FilterIndexer extends Indexer { this.put(layout.f.encode(hash), gcsFilter.hash()); } + /** + * save filter header + * @param {Hash} blockHash + * @param {Hash} filterHeader + * @param {Hash} filterHash + * @returns {Promise} + */ + + async saveFilterHeader(blockHash, filterHeader, filterHash) { + assert(blockHash); + assert(filterHeader); + assert(filterHash); + + const filter = new Filter(); + filter.header = filterHeader; + + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); + // console.log(layout.f.encode(blockHash)); + this.put(layout.f.encode(blockHash), filterHash); + } + + /** + * Save filter + * @param {Hash} blockHash + * @param {BasicFilter} basicFilter + * @param {Hash} filterHeader + * @returns {Promise} + */ + + async saveFilter(blockHash, basicFilter, filterHeader) { + assert(blockHash); + assert(basicFilter); + assert(filterHeader); + + const filter = new Filter(); + filter.filter = basicFilter.toRaw(); + filter.header = filterHeader; + + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); + // console.log(layout.f.encode(blockHash)); + this.put(layout.f.encode(blockHash), basicFilter.hash()); + } + /** * Prune compact filters. * @private diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js index 97d85f76b..b052d6a97 100644 --- a/lib/indexer/indexer.js +++ b/lib/indexer/indexer.js @@ -50,6 +50,8 @@ class Indexer extends EventEmitter { this.blocks = this.options.blocks; this.chain = this.options.chain; + this.neutrino = this.options.neutrino; + this.closing = false; this.db = null; this.batch = null; @@ -292,6 +294,11 @@ class Indexer extends EventEmitter { */ async _syncBlock(meta, block, view) { + if (this.neutrino) { + if (!this.batch) + this.start(); + return true; + } // In the case that the next block is being // connected or the current block disconnected // use the block and view being passed directly, @@ -636,6 +643,8 @@ class IndexOptions { this.cacheSize = 16 << 20; this.compression = true; + this.neutrino = false; + if (options) this.fromOptions(options); } @@ -697,6 +706,11 @@ class IndexOptions { this.compression = options.compression; } + if (options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + return this; } diff --git a/lib/net/peer.js b/lib/net/peer.js index dac2e265d..154c83e52 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1009,6 +1009,12 @@ class Peer extends EventEmitter { case packetTypes.GETHEADERS: this.request(packetTypes.HEADERS, timeout * 2); break; + case packetTypes.GETCFHEADERS: + this.request(packetTypes.CFHEADERS, timeout); + break; + case packetTypes.GETCFILTERS: + this.request(packetTypes.CFILTER, timeout); + break; case packetTypes.GETDATA: this.request(packetTypes.DATA, timeout * 2); break; @@ -1751,6 +1757,26 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * @param {Number} filterType - `0` = basic + * @param {Number} startHeight - Height to start at. + * @param {Hash} stopHash - Hash to stop at. + * @returns {void} + * @description Send `getcfilters` to peer. + */ + sendGetCFilters(filterType, startHeight, stopHash) { + const packet = new packets.GetCFiltersPacket( + filterType, + startHeight, + stopHash); + + this.logger.debug( + 'Sending getcfilters (type=%d, startHeight=%d, stopHash=%h).', + filterType, startHeight, stopHash); + + this.send(packet); + } + /** * Send `cfheaders` to peer. * @param {Number} filterType @@ -1773,6 +1799,27 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * @param {Number} filterType + * @param {Number} startHeight + * @param {Hash} stopHash + * @returns {void} + * @description Send `getcfheaders` to peer. + */ + + sendGetCFHeaders(filterType, startHeight, stopHash) { + const packet = new packets.GetCFHeadersPacket( + filterType, + startHeight, + stopHash); + + this.logger.debug( + 'Sending getcfheaders (type=%d, start=%h, stop=%h).', + filterType, startHeight, stopHash); + + this.send(packet); + } + /** * send `cfcheckpt` to peer. * @param {Number} filterType @@ -1793,6 +1840,25 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * Send `getcfcheckpt` to peer. + * @param {Number} filterType + * @param {Hash} stopHash + * @returns {void} + */ + + sendGetCFCheckpt(filterType, stopHash) { + const packet = new packets.GetCFCheckptPacket( + filterType, + stopHash); + + this.logger.debug( + 'Sending getcfcheckpt (type=%d, stop=%h).', + filterType, stopHash); + + this.send(packet); + } + /** * Send `mempool` to peer. */ diff --git a/lib/net/pool.js b/lib/net/pool.js index fcf11a3b8..ae8ae859b 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -35,6 +35,7 @@ const packetTypes = packets.types; const scores = HostList.scores; const {inspectSymbol} = require('../utils'); const {consensus} = require('../protocol'); +const BasicFilter = require('../golomb/basicFilter'); /** * Pool @@ -87,6 +88,10 @@ class Pool extends EventEmitter { this.hosts = new HostList(this.options); this.id = 0; + this.requestedFilterType = null; + this.getcfiltersStartHeight = null; + this.requestedStopHash = null; + if (this.options.spv) { this.spvFilter = BloomFilter.fromRate( 20000, 0.001, BloomFilter.flags.ALL); @@ -713,6 +718,57 @@ class Pool extends EventEmitter { this.compactBlocks.clear(); } + /** + * Start the filters headers sync. + */ + + async startFilterHeadersSync() { + this.logger.info('Starting filter headers sync (%s).', + this.chain.options.network); + if (!this.opened || !this.connected) + return; + + const cFHeaderHeight = await this.chain.getCFHeaderHeight(); + const startHeight = cFHeaderHeight + ? cFHeaderHeight + 1 : 1; + const chainHeight = await this.chain.height; + const stopHeight = chainHeight - startHeight + 1 > 2000 + ? 2000 : chainHeight; + const stopHash = await this.chain.getHash(stopHeight); + this.requestedFilterType = common.FILTERS.BASIC; + this.requestedStopHash = stopHash; + await this.peers.load.sendGetCFHeaders( + common.FILTERS.BASIC, + startHeight, + stopHash); + } + + /** + * Start the filters sync. + */ + + async startFilterSync() { + this.logger.info('Starting filter sync (%s).', + this.chain.options.network); + if (!this.opened || !this.connected) + return; + + const cFilterHeight = await this.chain.getCFilterHeight(); + const startHeight = cFilterHeight + ? cFilterHeight + 1 : 1; + const chainHeight = await this.chain.height; + const stopHeight = chainHeight - startHeight + 1 > 1000 + ? 1000 : chainHeight; + const stopHash = await this.chain.getHash(stopHeight); + this.requestedFilterType = common.FILTERS.BASIC; + this.getcfiltersStartHeight = startHeight; + this.requestedStopHash = stopHash; + await this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash); + } + /** * Start the headers sync using getHeaders messages. * @private @@ -1234,6 +1290,12 @@ class Pool extends EventEmitter { case packetTypes.GETCFCHECKPT: await this.handleGetCFCheckpt(peer, packet); break; + case packetTypes.CFCHECKPT: + await this.handleCFCheckpt(peer, packet); + break; + case packetTypes.CFHEADERS: + await this.handleCFHeaders(peer, packet); + break; case packetTypes.GETBLOCKS: await this.handleGetBlocks(peer, packet); break; @@ -1286,8 +1348,8 @@ class Pool extends EventEmitter { await this.handleBlockTxn(peer, packet); break; case packetTypes.CFILTER: - case packetTypes.CFHEADERS: - case packetTypes.CFCHECKPT: + await this.handleCFilters(peer, packet); + break; case packetTypes.UNKNOWN: await this.handleUnknown(peer, packet); break; @@ -1965,7 +2027,7 @@ class Pool extends EventEmitter { if (!stopHeight) return; - if (stopHeight - packet.startHeight >= common.MAX_CFILTERS) + if (stopHeight - packet.startHeight > common.MAX_CFILTERS) return; const indexer = this.getFilterIndexer(filtersByVal[packet.filterType]); @@ -2061,6 +2123,160 @@ class Pool extends EventEmitter { peer.sendCFCheckpt(packet.filterType, packet.stopHash, filterHeaders); } + /** + * Handle peer `CFCheckpt` packet. + * @method + * @private + * @param {Peer} peer + * @param {CFCheckptPacket} packet + */ + + async handleCFCheckpt(peer, packet) { + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + } + } + + /** + * Handle peer `CFHeaders` packet. + * @method + * @private + * @param {Peer} peer - Sender. + * @param {CFHeadersPacket} packet - Packet to handle. + * @returns {void} + */ + async handleCFHeaders(peer, packet) { + this.logger.info('Received CFHeaders packet from %s', peer.hostname()); + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + return; + } + + const filterType = packet.filterType; + + if (filterType !== this.requestedFilterType) { + this.logger.warning('Received CFHeaders packet with wrong filterType'); + peer.ban(); + peer.destroy(); + return; + } + + const stopHash = packet.stopHash; + if (!stopHash.equals(this.requestedStopHash)) { + this.logger.warning('Received CFHeaders packet with wrong stopHash'); + peer.ban(); + return; + } + let previousFilterHeader = packet.previousFilterHeader; + const filterHashes = packet.filterHashes; + let blockHeight = await this.chain.getHeight(stopHash) + - filterHashes.length + 1; + const stopHeight = await this.chain.getHeight(stopHash); + for (const filterHash of filterHashes) { + if (blockHeight > stopHeight) { + peer.ban(); + return; + } + const basicFilter = new BasicFilter(); + basicFilter._hash = filterHash; + const filterHeader = basicFilter.header(previousFilterHeader); + const lastFilterHeader = this.cfHeaderChain.tail; + const cfHeaderEntry = new CFHeaderEntry( + filterHash, lastFilterHeader.height + 1); + this.cfHeaderChain.push(cfHeaderEntry); + const blockHash = await this.chain.getHash(blockHeight); + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + await indexer.saveFilterHeader(blockHash, filterHeader, filterHash); + previousFilterHeader = filterHeader; + await this.chain.saveCFHeaderHeight(blockHeight); + blockHeight++; + const cFHeaderHeight = await this.chain.getCFHeaderHeight(); + this.logger.info('CFHeaderHeight: %d', cFHeaderHeight); + } + if (this.headerChain.tail.height <= stopHeight) + this.emit('cfheaders'); + else { + const nextStopHeight = stopHeight + 2000 < this.chain.height + ? stopHeight + 2000 : this.chain.height; + const nextStopHash = await this.chain.getHash(nextStopHeight); + this.requestedStopHash = nextStopHash; + peer.sendGetCFHeaders(filterType, stopHeight + 1, nextStopHash); + } + } + + async handleCFilters(peer, packet) { + this.logger.info('Received CFilter packet from %s', peer.hostname()); + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + return; + } + + const blockHash = packet.blockHash; + const filterType = packet.filterType; + const filter = packet.filterBytes; + + if (filterType !== this.requestedFilterType) { + this.logger.warning('Received CFilter packet with wrong filterType'); + peer.ban(); + peer.destroy(); + return; + } + + const blockHeight = await this.chain.getHeight(blockHash); + const stopHeight = await this.chain.getHeight(this.requestedStopHash); + + if (!(blockHeight >= this.getcfiltersStartHeight + && blockHeight <= stopHeight)) { + this.logger.warning('Received CFilter packet with wrong blockHeight'); + peer.ban(); + return; + } + + const basicFilter = new BasicFilter(); + const gcsFilter = basicFilter.fromNBytes(filter); + + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + const filterHeader = await indexer.getFilterHeader(blockHash); + await indexer.saveFilter(blockHash, gcsFilter, filterHeader); + + await this.chain.saveCFilterHeight(blockHeight); + const cFilterHeight = await this.chain.getCFilterHeight(); + this.logger.info('CFilter height: %d', cFilterHeight); + this.emit('cfilter', blockHash, gcsFilter); + const startHeight = stopHeight + 1; + let nextStopHeight; + if (cFilterHeight === stopHeight + && stopHeight < this.chain.height) { + if (startHeight + 1000 < this.chain.height) { + nextStopHeight = stopHeight + 1000; + const stopHash = await this.chain.getHash(nextStopHeight); + this.getcfiltersStartHeight = startHeight; + this.requestedStopHash = stopHash; + this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash + ); + } else { + nextStopHeight = this.chain.height; + const stopHash = await this.chain.getHash(nextStopHeight); + this.getcfiltersStartHeight = startHeight; + this.requestedStopHash = stopHash; + this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash + ); + return; + } + } else if (cFilterHeight === this.chain.height) { + this.chain.emit('full'); + } + } + /** * Handle `getblocks` packet. * @method diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index cd373d3b9..928a3756d 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -645,28 +645,6 @@ class FullNode extends Node { return false; } - - /** - * Retrieve compact filter by hash. - * @param {Hash | Number} hash - * @param {Number} type - * @returns {Promise} - Returns {@link Buffer}. - */ - - async getBlockFilter(hash, filterType) { - const Indexer = this.filterIndexers.get(filterType); - - if (!Indexer) - return null; - - if (typeof hash === 'number') - hash = await this.chain.getHash(hash); - - if (!hash) - return null; - - return Indexer.getFilter(hash); - } } /* diff --git a/lib/node/http.js b/lib/node/http.js index 8448ec015..36ea3eb92 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -291,7 +291,8 @@ class HTTP extends Server { enforce(hash != null, 'Hash or height required.'); - const filter = await this.node.getBlockFilter(hash); + const filterName = valid.str(1, 'BASIC').toUpperCase(); + const filter = await this.node.getBlockFilter(hash, filterName); if (!filter) { res.json(404); @@ -301,6 +302,24 @@ class HTTP extends Server { res.json(200, filter.toJSON()); }); + this.get('/filterheader/:block', async (req, res) => { + const valid = Validator.fromRequest(req); + const hash = valid.uintbrhash('block'); + + enforce(hash != null, 'Hash or height required.'); + + const filterName = valid.str(1, 'BASIC').toUpperCase(); + const filterHeader = await this.node. + getBlockFilterHeader(hash, filterName); + + if (!filterHeader) { + res.json(404); + return; + } + + res.json(200, filterHeader.toJSON()); + }); + // Mempool snapshot this.get('/mempool', async (req, res) => { enforce(this.mempool, 'No mempool available.'); diff --git a/lib/node/node.js b/lib/node/node.js index 6ef9803fe..e2e20f453 100644 --- a/lib/node/node.js +++ b/lib/node/node.js @@ -413,6 +413,50 @@ class Node extends EventEmitter { await plugin.close(); } } + + /** + * Retrieve compact filter by hash/height. + * @param {Hash | Number} hash + * @param {Number} type + * @returns {Promise} - Returns {@link Buffer}. + */ + + async getBlockFilter(hash, filterType) { + const Indexer = this.filterIndexers.get(filterType); + + if (!Indexer) + return null; + + if (typeof hash === 'number') + hash = await this.chain.getHash(hash); + + if (!hash) + return null; + + return Indexer.getFilter(hash); + } + + /** + * Retrieve compact filter header by hash/height. + * @param {Hash | Number} hash + * @param {Number} type + * @returns {Promise} - Returns {@link Buffer}. + */ + + async getBlockFilterHeader(hash, filterType) { + const Indexer = this.filterIndexers.get(filterType); + + if (!Indexer) + return null; + + if (typeof hash === 'number') + hash = await this.chain.getHash(hash); + + if (!hash) + return null; + + return Indexer.getFilterHeader(hash); + } } /* diff --git a/lib/node/rpc.js b/lib/node/rpc.js index ab67affb9..bc1e93d44 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -155,11 +155,14 @@ class RPC extends RPCBase { this.add('getblockchaininfo', this.getBlockchainInfo); this.add('getbestblockhash', this.getBestBlockHash); this.add('getblockcount', this.getBlockCount); + this.add('getfiltercount', this.getFilterCount); + this.add('getfilterheadercount', this.getFilterHeaderCount); this.add('getblock', this.getBlock); this.add('getblockbyheight', this.getBlockByHeight); this.add('getblockhash', this.getBlockHash); this.add('getblockheader', this.getBlockHeader); this.add('getblockfilter', this.getBlockFilter); + this.add('getblockfilterheader', this.getBlockFilterHeader); this.add('getchaintips', this.getChainTips); this.add('getdifficulty', this.getDifficulty); this.add('getmempoolancestors', this.getMempoolAncestors); @@ -629,6 +632,22 @@ class RPC extends RPCBase { return this.chain.tip.height; } + async getFilterCount(args, help) { + if (help || args.length !== 0) + throw new RPCError(errs.MISC_ERROR, 'getfiltercount'); + + const height = await this.chain.getCFilterHeight(); + return height; + } + + async getFilterHeaderCount(args, help) { + if (help || args.length !== 0) + throw new RPCError(errs.MISC_ERROR, 'getfilterheadercount'); + + const height = await this.chain.getCFHeaderHeight(); + return height; + } + async getBlock(args, help) { if (help || args.length < 1 || args.length > 3) throw new RPCError(errs.MISC_ERROR, 'getblock "hash" ( verbose )'); @@ -769,6 +788,32 @@ class RPC extends RPCBase { return filter.toJSON(); } + async getBlockFilterHeader(args, help) { + if (help || args.length < 1 || args.length > 2) { + throw new RPCError(errs.MISC_ERROR, + 'getblockfilterheader "hash" ( "type" )'); + } + + const valid = new Validator(args); + const hash = valid.brhash(0); + const filterName = valid.str(1, 'BASIC').toUpperCase(); + + const filterType = filters[filterName]; + + if (!hash) + throw new RPCError(errs.MISC_ERROR, 'Invalid block hash.'); + + if (!filterType) + throw new RPCError(errs.MISC_ERROR, 'Filter type not supported'); + + const filterHeader = await this.node.getBlockFilterHeader(hash, filterName); + + if (!filterHeader) + throw new RPCError(errs.MISC_ERROR, 'Block filter header not found.'); + + return filterHeader; + } + async getChainTips(args, help) { if (help || args.length !== 0) throw new RPCError(errs.MISC_ERROR, 'getchaintips'); diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 9f6c43600..9a9b1ee6b 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -37,6 +37,13 @@ class NodeClient extends AsyncEmitter { init() { this.node.chain.on('connect', async (entry, block) => { + if (!this.opened || this.node.neutrino) + return; + + await this.emitAsync('block connect', entry, block.txs); + }); + + this.node.chain.on('getblockpeer', async (entry, block) => { if (!this.opened) return; @@ -50,6 +57,13 @@ class NodeClient extends AsyncEmitter { await this.emitAsync('block disconnect', entry); }); + this.node.pool.on('cfilter', async (blockHeight, filter) => { + if (!this.opened) + return; + + await this.emitAsync('cfilter', blockHeight, filter); + }); + this.node.on('tx', (tx) => { if (!this.opened) return; From 059acb4438615f535440bfc6c78a13309e8f0f8a Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Thu, 20 Jul 2023 23:58:44 +0530 Subject: [PATCH 3/5] test --- test/neutrino-test.js | 97 ++++++++++++++++++++++++++++++++++++++ test/node-rpc-test.js | 10 ++++ test/p2p-bip157-test.js | 100 ++++++++++++++++++++++++++++++++++++++++ test/p2p-test.js | 55 +--------------------- 4 files changed, 208 insertions(+), 54 deletions(-) create mode 100644 test/neutrino-test.js create mode 100644 test/p2p-bip157-test.js diff --git a/test/neutrino-test.js b/test/neutrino-test.js new file mode 100644 index 000000000..2a0562a69 --- /dev/null +++ b/test/neutrino-test.js @@ -0,0 +1,97 @@ +'use strict'; + +const FullNode = require('../lib/node/fullnode'); +const NeutrinoNode = require('../lib/node/neutrino'); +const {forValue} = require('./util/common'); +const assert = require('bsert'); +describe('neutrino', function () { + this.timeout(100000); + + const node1 = new NeutrinoNode({ + network: 'regtest', + memory: true, + port: 10000, + httpPort: 20000, + neutrino: true, + logConsole: true, + logLevel: 'debug', + only: '127.0.0.1' + }); + + const node2 = new FullNode({ + network: 'regtest', + memory: true, + listen: true, + indexFilter: true, + bip157: true + }); + + async function mineBlocks(n) { + while (n) { + const block = await node2.miner.mineBlock(); + await node2.chain.add(block); + await new Promise(resolve => setTimeout(resolve, 20)); + n--; + } + await forValue(node1.chain, 'height', node2.chain.height); + } + + before(async function () { + const waitForConnection = new Promise((resolve, reject) => { + node1.pool.once('peer open', async (peer) => { + resolve(peer); + }); + }); + + await node1.open(); + await node2.open(); + await node1.connect(); + await node2.connect(); + node1.startSync(); + node2.startSync(); + await waitForConnection; + await mineBlocks(1000); + }); + + after(async () => { + await node1.close(); + await node2.close(); + }); + + describe('getheaders', () => { + it('should getheaders', async () => { + await mineBlocks(10); + assert.equal(node1.chain.height, node2.chain.height); + }); + }); + + describe('getcfheaders', () => { + it('should getcfheaders', async () => { + await new Promise(resolve => setTimeout(resolve, 400)); + const headerHeight = await node1.chain.getCFHeaderHeight(); + assert.equal(headerHeight, node1.chain.height); + }); + }); + + describe('getcfilters', () => { + it('should getcfilters', async () => { + await new Promise(resolve => setTimeout(resolve, 400)); + const filterHeight = await node1.chain.getCFilterHeight(); + assert.equal(filterHeight, node1.chain.height); + }); + }); + + describe('save filters', () => { + it('should save filters correctly', async () => { + const filterIndexer = node1.filterIndexers.get('BASIC'); + for (let i = 0; i < node1.chain.height; i++) { + const hash = await node1.chain.getHash(i); + const filterHeader = await filterIndexer.getFilterHeader(hash); + assert(filterHeader); + const filter = await filterIndexer.getFilter(hash); + assert(filter); + assert(filterHeader.equals(filter.header)); + } + }); + }); +}); diff --git a/test/node-rpc-test.js b/test/node-rpc-test.js index cdef5b4dd..b44fa0d27 100644 --- a/test/node-rpc-test.js +++ b/test/node-rpc-test.js @@ -189,6 +189,16 @@ describe('RPC', function() { assert.strictEqual(expected.filter, info.filter); }); + it('should rpc getblockfilterheader', async () => { + const hash = await nclient.execute('getblockhash', [node.chain.tip.height]); + const info = await nclient.execute('getblockfilterheader', [hash, 'BASIC']); + const indexer = node.filterIndexers.get('BASIC'); + const filterHeader = await indexer.getFilterHeader(node.chain.tip.hash); + const expected = filterHeader.toJSON(); + + assert.deepStrictEqual(expected, info); + }); + describe('Blockchain', function () { it('should rpc getchaintips', async () => { const info = await nclient.execute('getchaintips', []); diff --git a/test/p2p-bip157-test.js b/test/p2p-bip157-test.js new file mode 100644 index 000000000..b72c7175a --- /dev/null +++ b/test/p2p-bip157-test.js @@ -0,0 +1,100 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ + +'use strict'; + +const assert = require('bsert'); +const FullNode = require('../lib/node/fullnode'); +const NeutrinoNode = require('../lib/node/neutrino'); +const {forValue} = require('./util/common'); +const {MAX_CFILTERS} = require('../lib/net/common'); +const packets = require('../lib/net/packets'); + +describe('P2P', function () { + this.timeout(50000); + + const node1 = new NeutrinoNode({ + network: 'regtest', + memory: true, + port: 10000, + httpPort: 20000, + only: '127.0.0.1', + neutrino: true + }); + + const node2 = new FullNode({ + network: 'regtest', + memory: true, + listen: true, + indexFilter: true, + bip157: true + }); + + let peer; + const nodePackets = {}; + + node1.pool.on('packet', (packet) => { + if (!nodePackets[packet.cmd]) + nodePackets[packet.cmd] = [packet]; + else + nodePackets[packet.cmd].push(packet); + }); + + async function mineBlocks(n) { + while (n) { + const block = await node2.miner.mineBlock(); + await node2.chain.add(block); + await new Promise(resolve => setTimeout(resolve, 20)); + n--; + } + await forValue(node1.chain, 'height', node2.chain.height); + } + + before(async () => { + const waitForConnection = new Promise((resolve, reject) => { + node1.pool.once('peer open', async (peer) => { + resolve(peer); + }); + }); + + await node1.open(); + await node2.open(); + await node1.connect(); + await node2.connect(); + node1.startSync(); + node2.startSync(); + + // `peer` is node2, from node1's perspective. + // So peer.send() sends a packet from node1 to node2, + // and `nodePackets` catches the response packets that + // node2 sends back to node1. + peer = await waitForConnection; + }); + + after(async () => { + await node1.close(); + await node2.close(); + }); + + describe('BIP157', function () { + before(async () => { + // Do not exceed limit, including genesis block + await mineBlocks(MAX_CFILTERS - node1.chain.height - 1); + }); + + it('CFCheckpt', async () => { + nodePackets.cfcheckpt = []; + + await mineBlocks(2); + + const pkt = new packets.GetCFCheckptPacket( + 0, + node1.chain.tip.hash + ); + + peer.send(pkt); + await forValue(nodePackets.cfcheckpt, 'length', 1); + assert.strictEqual(nodePackets.cfcheckpt[0].filterHeaders.length, 1); + }); + }); +}); diff --git a/test/p2p-test.js b/test/p2p-test.js index 85eae0849..b9502550d 100644 --- a/test/p2p-test.js +++ b/test/p2p-test.js @@ -6,8 +6,6 @@ const assert = require('bsert'); const FullNode = require('../lib/node/fullnode'); const {forValue} = require('./util/common'); -const {MAX_CFILTERS} = require('../lib/net/common'); -const packets = require('../lib/net/packets'); describe('P2P', function () { this.timeout(5000); @@ -60,6 +58,7 @@ describe('P2P', function () { await node2.connect(); node1.startSync(); node2.startSync(); + await mineBlocks(1); // `peer` is node2, from node1's perspective. // So peer.send() sends a packet from node1 to node2, @@ -73,58 +72,6 @@ describe('P2P', function () { await node2.close(); }); - describe('BIP157', function () { - before(async () => { - // Do not exceed limit, including genesis block - await mineBlocks(MAX_CFILTERS - node1.chain.height - 1); - }); - - it('CFilters', async () => { - nodePackets.cfilter = []; - - const pkt = new packets.GetCFiltersPacket( - 0, - 0, - node1.chain.tip.hash - ); - - peer.send(pkt); - await forValue(nodePackets.cfilter, 'length', MAX_CFILTERS); - }); - - it('CFHeaders', async () => { - nodePackets.cfheaders = []; - - const pkt = new packets.GetCFHeadersPacket( - 0, - 0, - node1.chain.tip.hash - ); - - peer.send(pkt); - await forValue(nodePackets.cfheaders, 'length', 1); - assert.strictEqual( - nodePackets.cfheaders[0].filterHashes.length, - node1.chain.height + 1 - ); - }); - - it('CFCheckpt', async () => { - nodePackets.cfcheckpt = []; - - await mineBlocks(2); - - const pkt = new packets.GetCFCheckptPacket( - 0, - node1.chain.tip.hash - ); - - peer.send(pkt); - await forValue(nodePackets.cfcheckpt, 'length', 1); - assert.strictEqual(nodePackets.cfcheckpt[0].filterHeaders.length, 1); - }); - }); - describe('Compact Blocks', function () { it('should get compact block in low bandwidth mode', async () => { nodePackets.inv = []; From 9750e821afc994293f6919cf9a3c88e5cad23aa4 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Sun, 23 Jul 2023 02:08:55 +0530 Subject: [PATCH 4/5] fix Headers-Sync (maybeSync) --- lib/blockchain/chain.js | 12 +--- lib/net/pool.js | 6 +- lib/node/neutrino.js | 7 +- test/neutrino-test.js | 155 ++++++++++++++++++++++++++-------------- 4 files changed, 113 insertions(+), 67 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 6537e72ef..e0c3c5335 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2021,22 +2021,14 @@ class Chain extends AsyncEmitter { if (this.synced) return; - if (this.options.neutrino && this.getProgress() < 1) - return; - if (this.options.checkpoints) { + if (this.options.checkpoints) if (this.height < this.network.lastCheckpoint) return; - } else if (!this.options.neutrino && - this.tip.time < util.now() - this.network.block.maxTipAge) - return; if (!this.hasChainwork()) return; this.synced = true; - if (this.options.neutrino) - this.emit('headersFull'); - else - this.emit('full'); + this.emit('full'); } /** diff --git a/lib/net/pool.js b/lib/net/pool.js index ae8ae859b..439f5d561 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -898,7 +898,7 @@ class Pool extends EventEmitter { return false; // Ask for the mempool if we're synced. - if (this.network.requestMempool) { + if (this.network.requestMempool && !this.options.neutrino) { if (peer.loader && this.chain.synced) peer.sendMempool(); } @@ -1807,8 +1807,6 @@ class Pool extends EventEmitter { */ async handleTXInv(peer, hashes) { - if (this.options.neutrino) - return; assert(hashes.length > 0); if (this.syncing && !this.chain.synced) @@ -2273,7 +2271,7 @@ class Pool extends EventEmitter { return; } } else if (cFilterHeight === this.chain.height) { - this.chain.emit('full'); + this.emit('cfilters'); } } diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 8077943f2..7e1c4b3ed 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -161,7 +161,7 @@ class Neutrino extends Node { this.emit('reset', tip); }); - this.chain.on('headersFull', async () => { + this.chain.on('full', async () => { if (this.chain.height === 0) return; this.logger.info('Block Headers are fully synced'); @@ -175,6 +175,11 @@ class Neutrino extends Node { await this.pool.startFilterSync(); }); + this.pool.on('cfilters', async () => { + this.logger.info('Compact Filters are fully synced'); + this.startSync(); + }); + this.loadPlugins(); } diff --git a/test/neutrino-test.js b/test/neutrino-test.js index 2a0562a69..8053fd77b 100644 --- a/test/neutrino-test.js +++ b/test/neutrino-test.js @@ -1,24 +1,17 @@ 'use strict'; +const Network = require('../lib/protocol/network'); const FullNode = require('../lib/node/fullnode'); const NeutrinoNode = require('../lib/node/neutrino'); const {forValue} = require('./util/common'); const assert = require('bsert'); -describe('neutrino', function () { - this.timeout(100000); - const node1 = new NeutrinoNode({ - network: 'regtest', - memory: true, - port: 10000, - httpPort: 20000, - neutrino: true, - logConsole: true, - logLevel: 'debug', - only: '127.0.0.1' - }); +const network = Network.get('regtest'); - const node2 = new FullNode({ +describe('Neutrino', function () { + this.timeout(10000); + + const fullNode = new FullNode({ network: 'regtest', memory: true, listen: true, @@ -28,64 +21,70 @@ describe('neutrino', function () { async function mineBlocks(n) { while (n) { - const block = await node2.miner.mineBlock(); - await node2.chain.add(block); - await new Promise(resolve => setTimeout(resolve, 20)); + const block = await fullNode.miner.mineBlock(); + await fullNode.chain.add(block); n--; } - await forValue(node1.chain, 'height', node2.chain.height); } - before(async function () { - const waitForConnection = new Promise((resolve, reject) => { - node1.pool.once('peer open', async (peer) => { - resolve(peer); - }); - }); - - await node1.open(); - await node2.open(); - await node1.connect(); - await node2.connect(); - node1.startSync(); - node2.startSync(); - await waitForConnection; - await mineBlocks(1000); + before(async () => { + await fullNode.open(); + await fullNode.connect(); + await mineBlocks(200); }); after(async () => { - await node1.close(); - await node2.close(); + await fullNode.close(); }); - describe('getheaders', () => { - it('should getheaders', async () => { + describe('No Checkpoints', function () { + const neutrinoNode = new NeutrinoNode({ + network: 'regtest', + memory: true, + port: 10000, + httpPort: 20000, + neutrino: true, + only: '127.0.0.1' + }); + + before(async () => { + await neutrinoNode.open(); + await neutrinoNode.connect(); + assert.strictEqual(neutrinoNode.chain.height, 0); + assert(neutrinoNode.chain.synced); + }); + + after(async () => { + await neutrinoNode.close(); + }); + + it('should initial sync', async () => { + neutrinoNode.startSync(); + await forValue(neutrinoNode.chain, 'height', fullNode.chain.height); + }); + + it('should get new blocks headers-only', async () => { await mineBlocks(10); - assert.equal(node1.chain.height, node2.chain.height); + await new Promise(resolve => setTimeout(resolve, 400)); + assert.equal(neutrinoNode.chain.height, fullNode.chain.height); }); - }); - describe('getcfheaders', () => { it('should getcfheaders', async () => { - await new Promise(resolve => setTimeout(resolve, 400)); - const headerHeight = await node1.chain.getCFHeaderHeight(); - assert.equal(headerHeight, node1.chain.height); + await new Promise(resolve => setTimeout(resolve, 400)); + const headerHeight = await neutrinoNode.chain.getCFHeaderHeight(); + assert.equal(headerHeight, neutrinoNode.chain.height); }); - }); - describe('getcfilters', () => { it('should getcfilters', async () => { - await new Promise(resolve => setTimeout(resolve, 400)); - const filterHeight = await node1.chain.getCFilterHeight(); - assert.equal(filterHeight, node1.chain.height); + await new Promise(resolve => setTimeout(resolve, 400)); + const filterHeight = await neutrinoNode.chain.getCFilterHeight(); + assert.equal(filterHeight, neutrinoNode.chain.height); }); - }); - describe('save filters', () => { it('should save filters correctly', async () => { - const filterIndexer = node1.filterIndexers.get('BASIC'); - for (let i = 0; i < node1.chain.height; i++) { - const hash = await node1.chain.getHash(i); + const filterIndexer = neutrinoNode.filterIndexers.get('BASIC'); + for (let i = 0; i < neutrinoNode.chain.height; i++) { + const hash = await neutrinoNode.chain.getHash(i); const filterHeader = await filterIndexer.getFilterHeader(hash); assert(filterHeader); const filter = await filterIndexer.getFilter(hash); @@ -94,4 +93,56 @@ describe('neutrino', function () { } }); }); + + describe('With Checkpoints', function () { + const neutrinoNode = new NeutrinoNode({ + network: 'regtest', + memory: true, + port: 10000, + httpPort: 20000, + logConsole: true, + logLevel: 'debug', + neutrino: true, + only: '127.0.0.1' + }); + + before(async () => { + // Set a new checkpoint from live regtrest chain + const entry = await fullNode.chain.getEntry(fullNode.chain.tip.height - 20); + network.checkpointMap[entry.height] = entry.hash; + network.lastCheckpoint = entry.height; + network.init(); + + await neutrinoNode.open(); + await neutrinoNode.connect(); + assert.strictEqual(neutrinoNode.chain.height, 0); + assert(!neutrinoNode.chain.synced); + }); + + after(async () => { + await neutrinoNode.close(); + + // Restore defaults + network.checkpointMap = {}; + network.lastCheckpoint = 0; + }); + + it('should initial sync', async () => { + let full = false; + neutrinoNode.chain.on('full', () => { + full = true; + }); + + neutrinoNode.startSync(); + await forValue(neutrinoNode.chain, 'height', fullNode.chain.height); + assert(full); + assert(neutrinoNode.chain.synced); + }); + + it('should get new blocks headers-only', async () => { + await mineBlocks(10); + await new Promise(resolve => setTimeout(resolve, 400)); + assert.equal(neutrinoNode.chain.height, fullNode.chain.height); + }); + }); }); From d1d8e80deafe3662136a29b7c6757fd00f57c113 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Wed, 2 Aug 2023 00:44:40 +0530 Subject: [PATCH 5/5] fix: headers-sync, refactor and test bip157 --- bin/bcoin-cli | 20 - lib/blockchain/chain.js | 31 +- lib/blockchain/chaindb.js | 63 +-- lib/blockchain/layout.js | 3 - lib/client/node.js | 5 - lib/indexer/filterindexer.js | 28 +- lib/indexer/indexer.js | 7 + lib/net/netaddress.js | 3 +- lib/net/peer.js | 24 +- lib/net/pool.js | 195 ++++----- lib/net/seeds/main.js | 827 ++++------------------------------- lib/node/http.js | 18 - lib/node/neutrino.js | 19 +- lib/node/node.js | 22 - lib/node/rpc.js | 45 -- lib/protocol/networks.js | 2 +- test/neutrino-test.js | 42 +- test/node-rpc-test.js | 10 - test/p2p-bip157-test.js | 100 ----- test/p2p-test.js | 158 +++++-- 20 files changed, 334 insertions(+), 1288 deletions(-) delete mode 100644 test/p2p-bip157-test.js diff --git a/bin/bcoin-cli b/bin/bcoin-cli index 9ca39cdfe..b9136ab43 100755 --- a/bin/bcoin-cli +++ b/bin/bcoin-cli @@ -129,22 +129,6 @@ class CLI { this.log(filter); } - async getFilterHeader() { - let hash = this.config.str(0, ''); - - if (hash.length !== 64) - hash = parseInt(hash, 10); - - const filterHeader = await this.client.getFilterHeader(hash); - - if (!filterHeader) { - this.log('Filter header not found.'); - return; - } - - this.log(filterHeader); - } - async estimateFee() { const blocks = this.config.uint(0, 1); @@ -262,9 +246,6 @@ class CLI { case 'filter': await this.getFilter(); break; - case 'filterheader': - await this.getFilterHeader(); - break; case 'fee': await this.estimateFee(); break; @@ -282,7 +263,6 @@ class CLI { this.log(' $ coin [hash+index/address]: View coins.'); this.log(' $ fee [target]: Estimate smart fee.'); this.log(' $ filter [hash/height]: View filter.'); - this.log(' $ filterheader [hash/height]: View filter header.'); this.log(' $ header [hash/height]: View block header.'); this.log(' $ info: Get server info.'); this.log(' $ mempool: Get mempool snapshot.'); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index e0c3c5335..9cd0a312f 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -1791,24 +1791,6 @@ class Chain extends AsyncEmitter { return this.hasEntry(hash); } - async getCFHeaderHeight() { - return await this.db.getCFHeaderHeight(); - } - - async saveCFHeaderHeight(height) { - this.db.neutrinoState.headerHeight = height; - await this.db.saveNeutrinoState(); - } - - async getCFilterHeight() { - return await this.db.getCFilterHeight(); - } - - async saveCFilterHeight(height) { - this.db.neutrinoState.filterHeight = height; - await this.db.saveNeutrinoState(); - } - /** * Find the corresponding block entry by hash or height. * @param {Hash|Number} hash/height @@ -2021,12 +2003,17 @@ class Chain extends AsyncEmitter { if (this.synced) return; - if (this.options.checkpoints) + if (this.options.checkpoints) { if (this.height < this.network.lastCheckpoint) return; + } + + if (this.tip.time < util.now() - this.network.block.maxTipAge) + return; if (!this.hasChainwork()) return; + this.synced = true; this.emit('full'); } @@ -2629,7 +2616,6 @@ class ChainOptions { this.compression = true; this.spv = false; - this.neutrino = false; this.bip91 = false; this.bip148 = false; this.prune = false; @@ -2676,11 +2662,6 @@ class ChainOptions { this.spv = options.spv; } - if (options.neutrino != null) { - assert(typeof options.neutrino === 'boolean'); - this.neutrino = options.neutrino; - } - if (options.prefix != null) { assert(typeof options.prefix === 'string'); this.prefix = options.prefix; diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index c4d2e0671..cb91accaa 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -46,7 +46,6 @@ class ChainDB { this.state = new ChainState(); this.pending = null; this.current = null; - this.neutrinoState = null; this.cacheHash = new LRU(this.options.entryCache, null, BufferMap); this.cacheHeight = new LRU(this.options.entryCache); @@ -91,11 +90,6 @@ class ChainDB { this.logger.info('ChainDB successfully initialized.'); } - if (this.options.neutrino) { - if (!this.neutrinoState) - this.neutrinoState = await this.getNeutrinoState(); - } - this.logger.info( 'Chain State: hash=%h tx=%d coin=%d value=%s.', this.state.tip, @@ -1007,7 +1001,7 @@ class ChainDB { */ async getRawBlock(block) { - if (this.options.spv && !this.options.neutrino) + if (this.options.spv) return null; const hash = await this.getHash(block); @@ -1676,39 +1670,6 @@ class ChainDB { b.put(layout.O.encode(), flags.toRaw()); return b.write(); } - - /** - * Get Neutrino State - * @returns {Promise} - Returns neutrino state - */ - - async getNeutrinoState() { - const data = await this.db.get(layout.N.encode()); - if (!data) - return new NeutrinoState(); - return NeutrinoState.fromRaw(data); - } - - async getCFHeaderHeight() { - const state = await this.getNeutrinoState(); - return state.headerHeight; - } - - async getCFilterHeight() { - const state = await this.getNeutrinoState(); - return state.filterHeight; - } - - /** - * Save Neutrino State - * @returns {void} - */ - async saveNeutrinoState() { - const state = this.neutrinoState.toRaw(); - const b = this.db.batch(); - b.put(layout.N.encode(), state); - return b.write(); - } } /** @@ -1991,28 +1952,6 @@ function fromU32(num) { return data; } -class NeutrinoState { - constructor() { // TODO: do we add support for multiple filters? - this.headerHeight = 0; - this.filterHeight = 0; - } - - toRaw() { - const bw = bio.write(8); - bw.writeU32(this.headerHeight); - bw.writeU32(this.filterHeight); - return bw.render(); - } - - static fromRaw(data) { - const state = new NeutrinoState(); - const br = bio.read(data); - state.headerHeight = br.readU32(); - state.filterHeight = br.readU32(); - return state; - } -} - /* * Expose */ diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index 532ccb050..337f95900 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -14,7 +14,6 @@ const bdb = require('bdb'); * O -> chain options * R -> tip hash * D -> versionbits deployments - * N -> neutrino state * e[hash] -> entry * h[hash] -> height * H[height] -> hash @@ -34,8 +33,6 @@ const layout = { O: bdb.key('O'), R: bdb.key('R'), D: bdb.key('D'), - N: bdb.key('N'), - F: bdb.key('H', ['hash256']), e: bdb.key('e', ['hash256']), h: bdb.key('h', ['hash256']), H: bdb.key('H', ['uint32']), diff --git a/lib/client/node.js b/lib/client/node.js index 5df661dfa..dd717e4f2 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -169,11 +169,6 @@ class NodeClient extends Client { return this.get(`/filter/${block}`); } - getFilterHeader(block) { - assert(typeof block === 'string' || typeof block === 'number'); - return this.get(`/filterheader/${block}`); - } - getBlockPeer(hash) { return this.call('get block peer', hash); } diff --git a/lib/indexer/filterindexer.js b/lib/indexer/filterindexer.js index ae88af139..02a44fb90 100644 --- a/lib/indexer/filterindexer.js +++ b/lib/indexer/filterindexer.js @@ -85,27 +85,6 @@ class FilterIndexer extends Indexer { this.put(layout.f.encode(hash), gcsFilter.hash()); } - /** - * save filter header - * @param {Hash} blockHash - * @param {Hash} filterHeader - * @param {Hash} filterHash - * @returns {Promise} - */ - - async saveFilterHeader(blockHash, filterHeader, filterHash) { - assert(blockHash); - assert(filterHeader); - assert(filterHash); - - const filter = new Filter(); - filter.header = filterHeader; - - await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); - // console.log(layout.f.encode(blockHash)); - this.put(layout.f.encode(blockHash), filterHash); - } - /** * Save filter * @param {Hash} blockHash @@ -114,8 +93,9 @@ class FilterIndexer extends Indexer { * @returns {Promise} */ - async saveFilter(blockHash, basicFilter, filterHeader) { + async saveFilter(blockHash, blockHeight, basicFilter, filterHeader) { assert(blockHash); + assert(blockHeight); assert(basicFilter); assert(filterHeader); @@ -123,9 +103,11 @@ class FilterIndexer extends Indexer { filter.filter = basicFilter.toRaw(); filter.header = filterHeader; + this.batch = this.db.batch(); + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); - // console.log(layout.f.encode(blockHash)); this.put(layout.f.encode(blockHash), basicFilter.hash()); + await super.syncHeight(blockHash, blockHeight); } /** diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js index b052d6a97..c5852b7e3 100644 --- a/lib/indexer/indexer.js +++ b/lib/indexer/indexer.js @@ -198,6 +198,13 @@ class Indexer extends EventEmitter { this.height = 0; } + async syncHeight(hash, height) { + const meta = new BlockMeta(hash, height); + await this._setTip(meta); + await this.commit(); + this.height = height; + } + /** * Bind to chain events and save listeners for removal on close * @private diff --git a/lib/net/netaddress.js b/lib/net/netaddress.js index b4b00f26d..79384069b 100644 --- a/lib/net/netaddress.js +++ b/lib/net/netaddress.js @@ -478,7 +478,8 @@ class NetAddress { NetAddress.DEFAULT_SERVICES = 0 | common.services.NETWORK | common.services.WITNESS - | common.services.BLOOM; + | common.services.BLOOM + | common.services.NODE_COMPACT_FILTERS; /* * Expose diff --git a/lib/net/peer.js b/lib/net/peer.js index 154c83e52..4756a75bd 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1680,6 +1680,11 @@ class Peer extends EventEmitter { this.compactWitness = packet.version === 2; } + sendSendHeaders() { + const packet = new packets.SendHeadersPacket(); + this.send(packet); + } + /** * Send `getheaders` to peer. Note that unlike * `getblocks`, `getheaders` can have a null locator. @@ -1840,25 +1845,6 @@ class Peer extends EventEmitter { this.send(packet); } - /** - * Send `getcfcheckpt` to peer. - * @param {Number} filterType - * @param {Hash} stopHash - * @returns {void} - */ - - sendGetCFCheckpt(filterType, stopHash) { - const packet = new packets.GetCFCheckptPacket( - filterType, - stopHash); - - this.logger.debug( - 'Sending getcfcheckpt (type=%d, stop=%h).', - filterType, stopHash); - - this.send(packet); - } - /** * Send `mempool` to peer. */ diff --git a/lib/net/pool.js b/lib/net/pool.js index 439f5d561..d19145c9a 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -69,6 +69,7 @@ class Pool extends EventEmitter { this.connected = false; this.disconnecting = false; this.syncing = false; + this.filterSyncing = false; this.discovering = false; this.spvFilter = null; this.txFilter = null; @@ -204,14 +205,7 @@ class Pool extends EventEmitter { return this.disconnect(); } - /** - * Reset header chain. - */ - - resetChain() { - if (!this.options.checkpoints && !this.options.neutrino) - return; - + resetHeadersChain() { if (!this.options.neutrino) this.checkpoints = false; this.headerTip = null; @@ -219,12 +213,12 @@ class Pool extends EventEmitter { this.headerNext = null; const tip = this.chain.tip; + if (this.options.neutrino) { this.headerChain.push(new HeaderEntry(tip.hash, tip.height)); - this.cfHeaderChain = new List(); - this.cfHeaderChain.push(new CFHeaderEntry(consensus.ZERO_HASH, 0)); return; } + if (tip.height < this.network.lastCheckpoint) { this.checkpoints = true; this.headerTip = this.getNextTip(tip.height); @@ -235,6 +229,26 @@ class Pool extends EventEmitter { } } + resetCFHeadersChain() { + if (!this.options.neutrino) + return; + + this.cfHeaderChain = new List(); + this.cfHeaderChain.push(new CFHeaderEntry(consensus.ZERO_HASH, null, 0)); + } + + /** + * Reset header chain. + */ + + resetChain() { + if (!this.options.checkpoints && !this.options.neutrino) + return; + + this.resetHeadersChain(); + this.resetCFHeadersChain(); + } + /** * Connect to the network. * @method @@ -661,10 +675,7 @@ class Pool extends EventEmitter { return; this.syncing = true; - if (this.options.neutrino) { - this.startHeadersSync(); - } else - this.resync(false); + this.resync(false); } /** @@ -723,17 +734,18 @@ class Pool extends EventEmitter { */ async startFilterHeadersSync() { + this.filterSyncing = true; this.logger.info('Starting filter headers sync (%s).', this.chain.options.network); if (!this.opened || !this.connected) return; - const cFHeaderHeight = await this.chain.getCFHeaderHeight(); + const cFHeaderHeight = this.cfHeaderChain.tail.height; const startHeight = cFHeaderHeight ? cFHeaderHeight + 1 : 1; const chainHeight = await this.chain.height; const stopHeight = chainHeight - startHeight + 1 > 2000 - ? 2000 : chainHeight; + ? startHeight + 1999 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); this.requestedFilterType = common.FILTERS.BASIC; this.requestedStopHash = stopHash; @@ -753,12 +765,14 @@ class Pool extends EventEmitter { if (!this.opened || !this.connected) return; - const cFilterHeight = await this.chain.getCFilterHeight(); + const indexer = this.getFilterIndexer(filtersByVal[common.FILTERS.BASIC]); + + const cFilterHeight = indexer.height; const startHeight = cFilterHeight ? cFilterHeight + 1 : 1; const chainHeight = await this.chain.height; const stopHeight = chainHeight - startHeight + 1 > 1000 - ? 1000 : chainHeight; + ? startHeight + 999 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); this.requestedFilterType = common.FILTERS.BASIC; this.getcfiltersStartHeight = startHeight; @@ -769,32 +783,6 @@ class Pool extends EventEmitter { stopHash); } - /** - * Start the headers sync using getHeaders messages. - * @private - * @return {Promise} - */ - - async startHeadersSync() { - if (!this.syncing) - return; - let locator; - try { - locator = await this.chain.getLocator(); - } catch (e) { - this.emit('error', e); - return; - } - - const peer = this.peers.load; - if (!peer) { - this.logger.info('No loader peer.'); - return; - } - this.chain.synced = false; - peer.sendGetHeaders(locator); - } - /** * Send a sync to each peer. * @private @@ -907,8 +895,6 @@ class Pool extends EventEmitter { peer.blockTime = Date.now(); if (this.options.neutrino) { peer.sendGetHeaders(locator); - if (!this.syncing) - this.startFilterHeadersSync(); return true; } if (this.checkpoints) { @@ -956,8 +942,8 @@ class Pool extends EventEmitter { if (items.length === common.MAX_INV) break; } - - this.getBlock(peer, items); + if (!this.options.neutrino) + this.getBlock(peer, items); } /** @@ -1291,11 +1277,14 @@ class Pool extends EventEmitter { await this.handleGetCFCheckpt(peer, packet); break; case packetTypes.CFCHECKPT: - await this.handleCFCheckpt(peer, packet); + await this.handleUnknown(peer, packet); break; case packetTypes.CFHEADERS: await this.handleCFHeaders(peer, packet); break; + case packetTypes.CFILTER: + await this.handleCFilters(peer, packet); + break; case packetTypes.GETBLOCKS: await this.handleGetBlocks(peer, packet); break; @@ -1347,9 +1336,6 @@ class Pool extends EventEmitter { case packetTypes.BLOCKTXN: await this.handleBlockTxn(peer, packet); break; - case packetTypes.CFILTER: - await this.handleCFilters(peer, packet); - break; case packetTypes.UNKNOWN: await this.handleUnknown(peer, packet); break; @@ -1392,6 +1378,10 @@ class Pool extends EventEmitter { peer.send(new packets.AddrPacket([addr])); } + if (this.options.neutrino) { + peer.sendSendHeaders(); + } + // We want compact blocks! if (this.options.compact) peer.sendCompact(this.options.blockMode); @@ -1736,13 +1726,6 @@ class Pool extends EventEmitter { if (this.options.hasWitness() && !peer.hasWitness()) return; - if (this.options.neutrino) { - const filterHeight = await this.chain.getCFilterHeight(); - if (filterHeight === this.chain.height) - this.startSync(); - return; - } - // Request headers instead. if (this.checkpoints) return; @@ -2121,21 +2104,6 @@ class Pool extends EventEmitter { peer.sendCFCheckpt(packet.filterType, packet.stopHash, filterHeaders); } - /** - * Handle peer `CFCheckpt` packet. - * @method - * @private - * @param {Peer} peer - * @param {CFCheckptPacket} packet - */ - - async handleCFCheckpt(peer, packet) { - if (!this.options.neutrino) { - peer.ban(); - peer.destroy(); - } - } - /** * Handle peer `CFHeaders` packet. * @method @@ -2164,7 +2132,7 @@ class Pool extends EventEmitter { const stopHash = packet.stopHash; if (!stopHash.equals(this.requestedStopHash)) { this.logger.warning('Received CFHeaders packet with wrong stopHash'); - peer.ban(); + peer.increaseBan(10); return; } let previousFilterHeader = packet.previousFilterHeader; @@ -2174,25 +2142,21 @@ class Pool extends EventEmitter { const stopHeight = await this.chain.getHeight(stopHash); for (const filterHash of filterHashes) { if (blockHeight > stopHeight) { - peer.ban(); + peer.increaseBan(10); return; } const basicFilter = new BasicFilter(); basicFilter._hash = filterHash; const filterHeader = basicFilter.header(previousFilterHeader); const lastFilterHeader = this.cfHeaderChain.tail; - const cfHeaderEntry = new CFHeaderEntry( - filterHash, lastFilterHeader.height + 1); - this.cfHeaderChain.push(cfHeaderEntry); const blockHash = await this.chain.getHash(blockHeight); - const indexer = this.getFilterIndexer(filtersByVal[filterType]); - await indexer.saveFilterHeader(blockHash, filterHeader, filterHash); + const cfHeaderEntry = new CFHeaderEntry(blockHash, + filterHeader, lastFilterHeader.height + 1); + this.cfHeaderChain.push(cfHeaderEntry); previousFilterHeader = filterHeader; - await this.chain.saveCFHeaderHeight(blockHeight); blockHeight++; - const cFHeaderHeight = await this.chain.getCFHeaderHeight(); - this.logger.info('CFHeaderHeight: %d', cFHeaderHeight); } + this.logger.info('CFHeader height: %d', this.cfHeaderChain.tail.height); if (this.headerChain.tail.height <= stopHeight) this.emit('cfheaders'); else { @@ -2205,7 +2169,12 @@ class Pool extends EventEmitter { } async handleCFilters(peer, packet) { - this.logger.info('Received CFilter packet from %s', peer.hostname()); + const filterType = packet.filterType; + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + if (indexer.height % 100 === 0) + this.logger.debug( + 'Received CFilter 100 packets from %s', peer.hostname() + ); if (!this.options.neutrino) { peer.ban(); peer.destroy(); @@ -2213,7 +2182,7 @@ class Pool extends EventEmitter { } const blockHash = packet.blockHash; - const filterType = packet.filterType; + this.cfHeaderChain.shift(); const filter = packet.filterBytes; if (filterType !== this.requestedFilterType) { @@ -2229,20 +2198,18 @@ class Pool extends EventEmitter { if (!(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight)) { this.logger.warning('Received CFilter packet with wrong blockHeight'); - peer.ban(); + peer.increaseBan(10); return; } const basicFilter = new BasicFilter(); const gcsFilter = basicFilter.fromNBytes(filter); - const indexer = this.getFilterIndexer(filtersByVal[filterType]); - const filterHeader = await indexer.getFilterHeader(blockHash); - await indexer.saveFilter(blockHash, gcsFilter, filterHeader); - - await this.chain.saveCFilterHeight(blockHeight); - const cFilterHeight = await this.chain.getCFilterHeight(); - this.logger.info('CFilter height: %d', cFilterHeight); + const filterHeader = this.cfHeaderChain.head.header; + await indexer.saveFilter(blockHash, blockHeight, gcsFilter, filterHeader); + const cFilterHeight = await indexer.height; + if (cFilterHeight % 100 === 0) + this.logger.info('CFilter height: %d', cFilterHeight); this.emit('cfilter', blockHash, gcsFilter); const startHeight = stopHeight + 1; let nextStopHeight; @@ -2271,6 +2238,7 @@ class Pool extends EventEmitter { return; } } else if (cFilterHeight === this.chain.height) { + this.filterSyncing = false; this.emit('cfilters'); } } @@ -2406,10 +2374,9 @@ class Pool extends EventEmitter { const headers = packet.items; if (!this.checkpoints && !this.options.neutrino) - // todo add support for checkpoints return; - if (!this.syncing) + if (!this.syncing || this.filterSyncing) return; if (!peer.loader) @@ -2427,10 +2394,17 @@ class Pool extends EventEmitter { let checkpoint = false; let node = null; + let hash = null; for (const header of headers) { + hash = header.hash(); + + if (this.options.neutrino) { + await this._addBlock(peer, header, chainCommon.flags.VERIFY_POW); + continue; + } + const last = this.headerChain.tail; - const hash = header.hash(); const height = last.height + 1; if (!header.verify()) { @@ -2452,8 +2426,7 @@ class Pool extends EventEmitter { node = new HeaderEntry(hash, height); - if (!this.options.neutrino && node.height === this.headerTip.height) { - // todo add support for checkpoints + if (node.height === this.headerTip.height) { if (!node.hash.equals(this.headerTip.hash)) { this.logger.warning( 'Peer sent an invalid checkpoint (%s).', @@ -2468,8 +2441,6 @@ class Pool extends EventEmitter { this.headerNext = node; this.headerChain.push(node); - if (this.options.neutrino) - await this._addBlock(peer, header, chainCommon.flags.VERIFY_POW); } this.logger.debug( @@ -2477,24 +2448,22 @@ class Pool extends EventEmitter { headers.length, peer.hostname()); + this.emit('headers'); + // If we received a valid header // chain, consider this a "block". peer.blockTime = Date.now(); // Request the blocks we just added. - if (checkpoint && !this.options.neutrino) { + if (checkpoint) { this.headerChain.shift(); this.resolveHeaders(peer); return; } // Request more headers. - if (this.chain.synced) - return; - if (this.options.neutrino) - peer.sendGetHeaders([node.hash]); - else - peer.sendGetHeaders([node.hash], this.headerTip.hash); + if (this.checkpoints) + peer.sendGetHeaders([hash], this.headerTip.hash); } /** @@ -2624,8 +2593,8 @@ class Pool extends EventEmitter { } this.logStatus(block); - - await this.resolveChain(peer, hash); + if (!this.options.neutrino) + await this.resolveChain(peer, hash); } /** @@ -4236,6 +4205,7 @@ class PoolOptions { if (this.neutrino) { this.requiredServices |= common.services.NODE_COMPACT_FILTERS; + this.requiredServices |= common.services.WITNESS; this.checkpoints = true; this.compact = false; } @@ -4787,8 +4757,9 @@ class CFHeaderEntry { * @constructor */ - constructor(hash, height) { - this.hash = hash; + constructor(blockHash, header, height) { + this.blockHash = blockHash; + this.header = header; this.height = height; this.prev = null; this.next = null; diff --git a/lib/net/seeds/main.js b/lib/net/seeds/main.js index 7509c8eec..e57d4146c 100644 --- a/lib/net/seeds/main.js +++ b/lib/net/seeds/main.js @@ -1,756 +1,79 @@ 'use strict'; module.exports = [ - '2.24.141.73:8333', - '5.8.18.29:8333', - '5.43.228.99:8333', - '5.145.10.122:8333', - '5.166.35.47:8333', - '5.188.187.130:8333', - '5.199.133.193:8333', - '5.206.226.216:8333', - '5.206.226.231:8333', - '13.92.254.226:8335', - '13.125.188.128:8333', - '18.228.144.20:8333', - '23.175.0.200:8333', - '23.226.90.172:8333', - '23.233.107.28:8333', - '23.245.24.154:8333', - '24.121.16.35:8333', - '24.150.94.79:8333', - '24.188.200.170:8333', - '24.246.31.205:8333', - '27.102.102.157:8333', - '31.6.98.94:8333', - '31.20.226.115:8333', - '31.21.182.79:8333', - '31.43.140.190:8333', - '31.132.135.134:8333', - '31.173.48.61:8333', - '32.214.183.114:8333', - '34.231.234.150:8333', - '35.209.114.159:8333', - '35.213.18.190:8333', - '37.97.228.224:8333', - '37.116.95.41:8333', - '37.123.132.33:8333', - '37.133.140.169:8334', - '37.134.165.205:8333', - '37.191.253.125:8333', - '39.108.68.237:7781', - '40.78.19.149:8333', - '42.60.217.183:8333', - '43.229.132.102:8333', - '45.58.126.138:8333', - '46.28.132.34:8333', - '46.166.162.45:20001', - '46.166.176.137:8333', - '46.227.68.104:8333', - '46.227.68.105:8333', - '47.74.32.190:8885', - '47.89.19.134:30303', - '47.97.117.250:8333', - '50.2.13.166:8333', - '50.5.163.139:8333', - '50.34.65.217:8333', - '50.66.209.54:8333', - '50.67.179.36:8333', - '51.15.166.138:8333', - '51.15.239.164:8333', - '51.154.60.34:8333', - '51.154.136.60:8333', - '52.116.159.247:8333', - '54.167.232.37:8333', - '58.22.123.120:8333', - '58.158.0.86:8333', - '62.45.159.66:8333', - '62.75.191.166:8333', - '62.75.210.81:8333', - '62.97.244.242:8333', - '62.107.200.30:8333', - '62.138.0.217:8333', - '62.213.214.207:8333', - '64.98.18.21:8333', - '65.79.145.209:8333', - '66.151.242.154:8335', - '66.206.13.51:8333', - '66.248.206.86:8333', - '67.40.207.169:8333', - '67.149.252.79:8333', - '67.193.189.42:8333', - '67.210.228.203:8333', - '67.220.22.78:8333', - '67.222.131.151:8333', - '68.168.122.2:8333', - '68.202.128.19:8333', - '68.206.21.144:8333', - '69.30.215.42:8333', - '69.59.18.22:8333', - '69.70.170.178:8333', - '69.132.150.43:8333', - '69.145.122.160:8333', - '70.26.149.104:8333', - '70.51.142.43:8333', - '70.63.170.86:8333', - '71.57.73.173:8333', - '71.237.255.140:8333', - '72.24.235.10:8333', - '72.95.104.94:8333', - '72.231.187.25:8333', - '72.253.239.246:8333', - '74.78.140.178:8333', - '74.83.234.97:8333', - '74.84.128.158:9333', - '74.197.236.58:8333', - '74.208.94.172:8333', - '74.220.255.190:8333', - '75.101.96.6:8333', - '75.157.77.34:8333', - '76.93.183.209:8333', - '76.174.129.203:8333', - '77.53.158.137:8333', - '77.85.204.149:8333', - '77.120.119.27:8433', - '77.134.172.81:8333', - '78.42.12.201:8333', - '78.58.140.102:8333', - '78.108.108.162:8333', - '78.119.180.62:8333', - '78.128.62.52:8333', - '78.130.148.218:8885', - '78.130.161.76:8333', - '78.143.214.223:8333', - '79.77.33.128:8333', - '79.175.125.210:8333', - '79.175.154.228:8333', - '80.79.114.34:8333', - '80.89.203.172:8001', - '80.100.128.128:8333', - '80.122.43.78:8333', - '80.151.124.127:8333', - '80.167.79.174:8333', - '80.211.191.11:8333', - '80.229.151.187:8333', - '81.4.102.69:8333', - '81.4.102.91:8333', - '81.6.34.154:8333', - '81.7.16.182:8333', - '81.7.17.202:8333', - '81.25.71.68:8444', - '81.235.185.150:8333', - '82.23.106.56:8333', - '82.29.58.109:8333', - '82.117.166.77:8333', - '82.145.41.24:8333', - '82.146.153.130:8333', - '82.149.97.25:17567', - '82.150.180.30:8333', - '82.177.176.24:8333', - '82.194.153.233:8333', - '82.197.215.125:8333', - '82.197.218.97:8333', - '82.199.102.133:8333', - '82.200.205.30:8333', - '82.221.111.136:8333', - '83.32.70.197:8333', - '83.58.134.138:8333', - '83.85.131.168:8333', - '83.163.211.75:8333', - '83.208.254.182:8333', - '83.243.191.199:8333', - '84.46.116.71:8333', - '84.52.255.147:8333', - '84.56.105.17:8333', - '84.59.243.22:8333', - '84.197.198.167:8333', - '84.214.74.65:8333', - '84.217.160.164:8333', - '84.227.14.62:8333', - '84.246.200.122:8333', - '85.14.79.26:8333', - '85.119.83.25:8333', - '85.190.0.5:8333', - '85.192.173.14:8333', - '85.214.80.203:8333', - '85.214.204.63:8333', - '85.229.166.15:8333', - '85.233.38.5:8333', - '86.76.7.132:8333', - '86.80.62.194:8333', - '86.107.204.50:8333', - '86.139.248.102:8333', - '87.79.68.86:8333', - '87.79.94.221:8333', - '87.99.79.123:8333', - '87.104.127.153:8333', - '87.117.19.226:8333', - '87.120.8.5:20008', - '87.224.163.66:8333', - '87.233.181.146:8333', - '87.249.207.89:8333', - '88.86.116.140:8333', - '88.86.116.141:8333', - '88.86.243.241:8333', - '88.87.93.52:1691', - '88.98.198.130:8333', - '88.99.109.66:8333', - '88.119.128.36:8333', - '88.129.253.46:8333', - '88.212.44.33:8333', - '89.23.35.9:8333', - '89.47.217.222:8333', - '89.106.199.38:8333', - '89.142.75.60:8333', - '89.179.126.97:8333', - '89.212.9.96:8333', - '89.218.198.46:8333', - '89.230.96.42:8333', - '90.125.157.153:8333', - '90.146.97.100:8333', - '90.182.165.18:8333', - '90.227.130.6:8333', - '91.92.128.32:8333', - '91.123.82.15:8333', - '91.135.0.187:8333', - '91.152.121.138:8333', - '91.178.131.108:8333', - '91.185.198.234:8333', - '91.193.237.88:8333', - '91.202.133.75:8885', - '91.204.99.178:8333', - '91.204.149.5:8333', - '91.216.149.28:8333', - '91.219.25.232:8333', - '91.222.128.59:8333', - '92.62.231.253:8333', - '92.63.192.206:8333', - '92.63.197.243:8333', - '92.63.197.245:8333', - '92.119.112.59:8333', - '92.243.244.101:8333', - '92.255.176.109:8333', - '93.38.119.141:8333', - '93.50.177.66:8333', - '93.79.204.222:10333', - '93.115.28.30:11100', - '93.115.89.76:8333', - '93.115.240.26:8333', - '93.123.180.164:8333', - '93.126.94.192:8333', - '93.170.128.106:8333', - '93.185.103.70:8333', - '93.189.145.169:8333', - '93.190.142.127:8333', - '93.228.3.234:8333', - '94.19.128.204:8333', - '94.26.49.71:8333', - '94.63.65.127:8333', - '94.72.143.28:8333', - '94.104.217.250:8333', - '94.209.115.52:8333', - '94.237.72.166:8333', - '94.242.255.31:8333', - '95.24.48.84:15426', - '95.69.249.63:8333', - '95.79.35.133:8333', - '95.87.226.56:8333', - '95.91.80.140:8333', - '95.102.60.168:8333', - '95.154.90.99:8333', - '95.156.252.34:8333', - '95.165.175.75:8333', - '95.174.125.24:18333', - '95.183.54.101:12853', - '95.211.189.3:8333', - '95.213.143.13:8333', - '95.213.184.109:778', - '96.9.80.109:8333', - '96.47.122.171:8333', - '97.81.244.191:8333', - '97.99.13.150:8333', - '97.104.206.3:8333', - '98.116.105.49:8333', - '99.224.131.4:8333', - '101.92.39.116:8333', - '101.100.163.118:8327', - '101.100.174.24:8333', - '101.251.68.146:12337', - '102.132.229.253:8333', - '103.14.244.190:8333', - '103.16.128.63:8333', - '103.59.144.135:8333', - '103.59.144.238:8333', - '103.99.168.100:8333', - '103.99.168.130:8333', - '103.100.220.46:8333', - '103.105.56.82:8333', - '103.106.208.207:8333', - '103.106.211.107:8333', - '103.108.228.51:8333', - '104.11.144.71:8333', - '104.128.228.252:8333', - '104.152.204.204:8333', - '104.153.30.236:8333', - '104.155.233.13:8333', - '104.198.126.116:8333', - '104.245.125.251:8333', - '106.12.57.72:8333', - '106.72.36.96:46289', - '106.163.158.127:8333', - '107.150.41.179:8333', - '107.191.116.103:8333', - '108.15.243.207:8333', - '108.58.252.82:8333', - '108.160.202.208:8333', - '108.213.205.103:8333', - '109.72.83.127:8333', - '109.99.63.159:8333', - '109.104.8.48:8333', - '109.183.251.77:8333', - '109.198.191.22:8333', - '109.236.90.122:58333', - '109.238.81.82:8333', - '109.248.206.13:8333', - '109.252.133.57:8333', - '111.90.145.57:8333', - '111.90.159.184:50001', - '113.35.179.149:8333', - '113.52.135.125:8333', - '115.47.141.250:8885', - '115.70.110.4:8333', - '116.58.171.67:8333', - '118.1.96.81:8333', - '118.103.126.140:28333', - '119.29.54.159:8333', - '119.207.78.152:8333', - '121.211.151.99:8333', - '122.112.148.153:8339', - '124.160.119.93:8333', - '128.197.128.222:8333', - '129.13.189.212:8333', - '129.97.243.18:8333', - '130.185.77.105:8333', - '130.255.187.86:8333', - '131.114.10.236:8333', - '131.188.40.34:8333', - '132.249.239.163:8333', - '133.18.1.114:8333', - '134.19.186.195:8333', - '136.36.123.20:8333', - '136.56.42.119:8333', - '137.226.34.46:8333', - '138.68.20.137:8333', - '141.101.8.36:8333', - '145.239.9.3:8333', - '145.249.106.103:8333', - '146.255.227.182:4033', - '147.192.18.175:8333', - '147.253.54.26:8333', - '148.66.58.58:8333', - '148.70.82.85:8333', - '149.90.34.119:8333', - '150.143.231.72:8333', - '153.92.127.216:8333', - '153.120.115.15:8333', - '153.124.187.220:8333', - '154.209.1.138:8333', - '154.211.159.200:8333', - '155.4.52.45:8333', - '156.19.19.90:8333', - '157.7.211.107:8333', - '159.100.248.234:8333', - '159.138.45.220:22235', - '160.16.0.30:8333', - '162.154.207.147:8333', - '163.158.243.230:8333', - '166.62.82.103:32771', - '166.62.100.55:8333', - '167.179.136.11:8333', - '168.235.74.110:8333', - '169.55.182.185:8333', - '171.33.177.9:8333', - '172.99.120.113:8333', - '172.105.112.233:8333', - '172.110.30.81:8333', - '173.21.218.95:8333', - '173.23.103.30:8000', - '173.51.177.2:8333', - '173.89.28.137:8333', - '173.208.128.10:8333', - '173.249.11.207:18333', - '174.65.135.60:8333', - '176.38.7.43:8333', - '176.92.150.12:8333', - '176.99.2.207:8333', - '176.126.167.10:8333', - '176.212.185.153:8333', - '176.223.136.171:8333', - '177.52.173.62:8333', - '178.33.136.162:8333', - '178.128.39.110:8333', - '178.143.50.8:8333', - '178.198.60.155:8333', - '178.236.137.63:8333', - '179.48.251.41:8333', - '180.150.52.37:8333', - '183.230.93.139:8333', - '184.80.255.250:8333', - '184.95.58.166:8336', - '184.180.129.98:8333', - '185.19.28.195:8333', - '185.25.48.184:8333', - '185.25.60.199:8333', - '185.50.68.64:8333', - '185.53.158.12:8333', - '185.61.79.213:8333', - '185.64.116.15:8333', - '185.95.219.53:8333', - '185.130.215.73:8333', - '185.130.215.187:8333', - '185.141.60.127:8333', - '185.147.11.108:8333', - '185.154.159.164:9992', - '185.198.56.77:8333', - '185.198.59.183:8333', - '185.216.140.33:8333', - '185.217.241.142:8333', - '185.249.199.106:8333', - '188.42.40.234:18333', - '188.65.212.138:8333', - '188.65.212.211:8333', - '188.68.45.143:8333', - '188.120.246.125:8333', - '188.134.5.47:8333', - '188.134.6.84:8333', - '188.167.101.51:8333', - '188.175.77.16:8333', - '188.213.168.152:8333', - '188.230.245.188:8333', - '189.121.185.148:8333', - '190.104.249.44:8333', - '190.184.198.34:8333', - '190.210.234.38:8333', - '190.218.190.85:8333', - '192.3.11.20:8333', - '192.3.11.24:8333', - '192.166.47.32:8333', - '192.167.149.143:8333', - '192.169.94.29:8333', - '192.169.94.70:8333', - '192.198.90.98:8333', - '192.254.89.134:8333', - '192.254.89.220:8333', - '193.41.78.125:8333', - '193.46.83.8:8333', - '193.59.41.11:8333', - '193.77.135.181:8333', - '193.84.116.22:8333', - '193.194.163.53:8333', - '194.71.225.55:8333', - '194.135.135.69:8333', - '194.158.92.150:8333', - '195.13.220.165:8333', - '195.56.63.10:8333', - '195.135.194.8:8333', - '195.168.36.20:8333', - '195.201.33.0:8333', - '195.202.169.149:8333', - '195.242.93.189:8333', - '198.1.231.6:8333', - '198.44.231.160:6333', - '198.54.113.59:8333', - '198.251.83.19:8333', - '199.68.199.4:8333', - '199.247.1.117:8333', - '199.247.10.26:8333', - '200.76.194.7:8333', - '201.241.2.85:8333', - '202.185.45.110:8333', - '203.86.207.53:8333', - '203.130.48.117:8885', - '204.14.245.180:8333', - '204.111.241.195:8333', - '204.152.203.98:8333', - '205.185.122.150:8333', - '206.124.149.66:8333', - '207.182.154.178:8333', - '208.81.1.105:8333', - '209.133.201.114:8333', - '209.173.25.140:8333', - '209.180.174.200:8333', - '209.190.36.13:8333', - '210.54.38.227:8333', - '210.54.39.99:8333', - '210.203.222.52:8223', - '211.104.154.140:8333', - '212.24.103.20:8333', - '212.33.204.190:8333', - '212.51.156.139:8333', - '212.109.198.126:8333', - '212.237.96.98:8333', - '212.241.70.213:8333', - '213.37.92.163:8333', - '213.89.98.199:8333', - '213.89.150.13:8333', - '213.174.156.72:8333', - '213.209.123.165:8333', - '213.227.152.108:8333', - '216.38.129.164:8333', - '216.86.154.215:8333', - '216.93.139.63:8333', - '216.186.250.53:8333', - '216.194.165.98:8333', - '217.22.132.220:8333', - '217.43.72.105:8333', - '217.64.47.138:8333', - '217.69.145.234:8333', - '217.158.9.102:8333', - '220.130.142.178:33389', - '220.233.138.130:8333', - '[2001:1ba8:401:32:b842:3891:5915:c68f]:8333', - '[2001:1bc0:cc::a001]:8333', - '[2001:250:200:7:d6a9:fcf4:e78d:2d82]:8333', - '[2001:4128:6135:e001:5054:ff:fe37:e9eb]:8333', - '[2001:41d0:fc63:9c00:1acc:d22f:3f5c:ef7f]:8333', - '[2001:44b8:4195:1801:5c73:5d67:d2a6:9910]:8333', - '[2001:4800:7821:101:be76:4eff:fe04:9f50]:8333', - '[2001:4801:7819:74:b745:b9d5:ff10:a61a]:8333', - '[2001:4801:7821:77:be76:4eff:fe10:c7f6]:8333', - '[2001:48d0:1:2163:0:ff:febe:5a80]:8333', - '[2001:48f8:1003::3ba]:8333', - '[2001:4ba0:fffa:5d::93]:8333', - '[2001:4c48:2:a328:d8a7:e0ff:fe96:403a]:8333', - '[2001:56b:dda9:4b00:49f9:121b:aa9e:de30]:8333', - '[2001:638:a000:4140::ffff:191]:8333', - '[2001:678:7dc:8::2]:8333', - '[2001:678:ec:1:250:56ff:fea7:47e9]:8333', - '[2001:67c:16dc:1201:5054:ff:fe17:4dac]:8333', - '[2001:67c:21ec:1000::a]:8333', - '[2001:67c:22fc:1337::5]:8333', - '[2001:67c:2824:8001:225:90ff:fe67:9830]:7777', - '[2001:67c:2b5c:101:216:3eff:fea3:5234]:8333', - '[2001:67c:2db8:13::83]:8333', - '[2001:718:801:311:5054:ff:fe19:c483]:8333', - '[2001:8003:d136:1001::11:ffd1]:8333', - '[2001:8d8:96a:9300::ad:ae2c]:8333', - '[2001:8f1:1602:700:1b28:a3e3:bb08:a708]:9444', - '[2001:8f8:1327:1587:3f10:5ab:804d:4039]:8333', - '[2001:ba8:1f1:f069::2]:8333', - '[2001:e42:103:100::30]:8333', - '[2400:2650:480:bc00:bcaf:7c49:8c9e:7cdf]:8333', - '[2400:4052:e20:4f00:69fe:bb33:7b1c:a1ca]:8333', - '[2400:8902::f03c:91ff:fea5:ebb7]:8333', - '[2401:1800:7800:102:be76:4eff:fe1c:a7d]:8333', - '[2401:2500:203:184::15]:8333', - '[2401:3900:2:1::2]:8333', - '[2402:7340:1:56::d0d]:8333', - '[2405:9800:ba01:251a:c53c:b80a:320d:5b41]:8333', - '[2405:aa00:2::40]:8333', - '[2409:10:ca20:1df0:224:e8ff:fe1f:60d9]:8333', - '[2409:13:1200:d200:16da:e9ff:fee9:b19a]:8333', - '[240d:1a:3c0:ab00:e9f1:87c:93ac:7687]:8333', - '[2602:ffc5:1f::1f:9211]:8333', - '[2604:2000:ffc0:0:5862:b6f8:fe72:762f]:8333', - '[2604:4300:a:2e:21b:21ff:fe11:392]:8333', - '[2604:5500:c2a3:7b00:cc6:373b:44a8:caa4]:8333', - '[2605:9880:201:17::4b7c]:8333', - '[2605:ae00:203::203]:8333', - '[2605:c000:2a0a:1::102]:8333', - '[2605:f700:100:400::131:5b54]:8333', - '[2606:c680:0:b:3830:34ff:fe66:6663]:8333', - '[2607:9280:b:73b:250:56ff:fe21:bf32]:8333', - '[2607:f128:40:1703::2]:8333', - '[2607:f3a0:1000:9:f82a:fdff:fea1:3315]:8333', - '[2607:f470:8:1048:ae1f:6bff:fe68:5e42]:8333', - '[2607:fd70:4a:babe:b00b:1e5:1bd5:f78]:8333', - '[2607:ff50:0:71::13]:8333', - '[2620:6e:a000:1:42:42:42:42]:8333', - '[2804:14d:baa7:9674:3615:9eff:fe23:d610]:8333', - '[2a00:1328:e101:c00::163]:8333', - '[2a00:1398:4:2a03:215:5dff:fed6:1033]:8333', - '[2a00:13a0:3015:1:85:14:79:26]:8333', - '[2a00:1630:14::101]:8333', - '[2a00:1768:2001:27::ef6a]:8333', - '[2a00:1828:a004:2::666]:8333', - '[2a00:1838:36:2c::3e95]:8333', - '[2a00:1b60:2:4:40d0:eff:fe88:ebd4]:8333', - '[2a00:7b80:452:2000::138]:8333', - '[2a00:7b80:454:2000::101]:8333', - '[2a00:8a60:e012:a00::21]:8333', - '[2a01:4240:5f52:9246::1]:8333', - '[2a01:430:17:1::ffff:1153]:8333', - '[2a01:488:66:1000:53a9:1573:0:1]:8333', - '[2a01:6f0:ffff:120::8dcb]:8333', - '[2a01:7a0:2:137a::11]:8333', - '[2a01:7a7:2:131b:20c:29ff:fe9a:3922]:8333', - '[2a01:7c8:d002:318:5054:ff:febe:cbb1]:8333', - '[2a01:cb00:d3d:7700:227:eff:fe28:c565]:8333', - '[2a01:d0:ffff:7368::2]:8333', - '[2a01:e0a:182:1300:591e:529:b376:c654]:8333', - '[2a01:e34:ee6b:2ab0:88c2:1c12:f4eb:c26c]:8333', - '[2a02:1205:34c3:d890:c0e:741e:c45f:3605]:8333', - '[2a02:2c8:1:400:34::184]:8333', - '[2a02:2f0d:202:f900:5e9a:d8ff:fe57:8bc5]:8333', - '[2a02:390:9000:0:218:7dff:fe10:be33]:8333', - '[2a02:4780:9:0:2:f928:f280:9a6f]:8333', - '[2a02:578:4f07:24:76ad:cef7:93c1:b9b9]:8333', - '[2a02:7aa0:1619::590:eba2]:8333', - '[2a02:7aa0:1619::adc:8de0]:8333', - '[2a02:8108:95bf:eae3:211:32ff:fe8e:b5b8]:8333', - '[2a02:c207:2014:9913::1]:18333', - '[2a02:e00:fff0:23f::1]:8333', - '[2a02:f680:1:1100::5453]:8333', - '[2a03:1b20:1:f410:40::3e]:16463', - '[2a03:2260:11e:301::8]:8333', - '[2a03:2260:11e:302::3]:8333', - '[2a03:4000:6:416c::43]:8333', - '[2a04:2180:1:c:f000::15]:8333', - '[2a04:3543:1000:2310:8492:b8ff:fe91:22e8]:8333', - '[2a05:6d40:b94e:d100:225:90ff:fe0d:cfc2]:8333', - '[2a05:fc87:4::6]:8333', - '[2a07:7200:ffff:c53f::e1:17]:8333', - '[2a0b:2ac0:1:0:d6ae:52ff:fe7b:741c]:8333', - '[2a0b:2ac0:1:0:d6ae:52ff:fe7b:88eb]:8333', - '25lhwv6jaqbtek5x.onion:8333', - '2empatdfea6vwete.onion:8333', - '2hpjn6ndxjafgoej.onion:8333', - '34aqcwnnuiqh234f.onion:8333', - '3frtobxxkgkhwjx7.onion:8333', - '3gxqibajrtysyp5o.onion:8333', - '3lf37sdzhpxh6fpv.onion:8333', - '3q5iydjrrutqjb2y.onion:8333', - '3qzrkpxduf44jqg5.onion:8333', - '3sami4tg4yhctjyc.onion:8333', - '3w77hrilg6q64opl.onion:8333', - '46xh2sbjsjiyl4fu.onion:8333', - '4ee44qsamrjpywju.onion:8333', - '4gwvtoppsaffaxg7.onion:8333', - '4haplrtkprjqhm2j.onion:8333', - '4u3y3zf2emynt6ui.onion:8333', - '4wx34hn3kybujklg.onion:8333', - '56czufbruq46sb2c.onion:8333', - '57dytizbai7o4kq7.onion:8333', - '5guaeulc7xm4g2mm.onion:8334', - '5mtvd4dk62ccdk4v.onion:8333', - '5nsfm4nqqzzprjrp.onion:8333', - '5pmjz6mmikyabaw5.onion:8333', - '6eurcxoqsa4qpiqq.onion:8333', - '6ivvkeseojsmpby4.onion:8333', - '6luc7owlbbaj52lr.onion:8333', - '6tlha6njtcuwpfa3.onion:8333', - '6ymgbvnn6d5nfmv4.onion:8333', - '6z5cyaswulhxcvhj.onion:8333', - '72y2n5rary4mywkz.onion:8333', - '7a354g25lnvry4ic.onion:8333', - '7b75ub5dapphemit.onion:8333', - '7xaqpr7exrtlnjbb.onion:8333', - 'a64haiqsl76l25gv.onion:8333', - 'ab7ftdfw6qhdx3re.onion:8333', - 'aiupgbtdqpmwfpuz.onion:8333', - 'akeg56rzkg7rsyyg.onion:8333', - 'akinbo7tlegsnsxn.onion:8333', - 'anem5aq4cr2zl7tz.onion:8333', - 'at3w5qisczgguije.onion:8333', - 'auo4zjsp44vydv6c.onion:8333', - 'b6vrxhrrle7jxiua.onion:8333', - 'bitcoinranliixsu.onion:8333', - 'blcktrgve5vetjsk.onion:8333', - 'bowg4prf63givea4.onion:8333', - 'cj2nexmwocyy5unq.onion:8333', - 'cjuek22p4vv4hzbu.onion:8333', - 'cklaa2xdawrb75fg.onion:8333', - 'coxiru76nnfw3vdj.onion:8333', - 'cqwcyvvk5xnqv3yw.onion:8333', - 'cwq2fuc54mlp3ojc.onion:8333', - 'dganr7dffsacayml.onion:8333', - 'djbsspmvlc6ijiis.onion:8333', - 'dmfwov5ycnpvulij.onion:8333', - 'dp2ekfbxubpdfrt4.onion:8333', - 'dw2ufbybrgtzssts.onion:4333', - 'dxv5u4xaeydpbrrp.onion:8333', - 'edkmfeaapvavhtku.onion:8333', - 'ejdoey3uay3cz7bs.onion:8333', - 'eladlvwflaahxomr.onion:8333', - 'ffhx6ttq7ejbodua.onion:8333', - 'fqdzxl4kjboae35b.onion:8333', - 'hbnnzteon75un65y.onion:8333', - 'hcyxhownxdv7yybw.onion:8333', - 'hdfcxll2tqs2l4jc.onion:8333', - 'hdld2bxyvzy45ds4.onion:8333', - 'hnqwmqikfmnkpdja.onion:8333', - 'hvmjovdasoin43wn.onion:8333', - 'hwzcbnenp6dsp6ow.onion:8333', - 'hz26wamjlbd7arrl.onion:8333', - 'i5ellwzndjuke242.onion:8333', - 'iapvpwzs4gpbl6fk.onion:8885', - 'if7fsvgyqwowxkcn.onion:8333', - 'ilukzjazxlxrbuwy.onion:8333', - 'ju5duo3r6p6diznc.onion:8333', - 'k3i3suxlg4w27uas.onion:8333', - 'k7omfveynnjg674e.onion:8333', - 'ko37ti7twplktxqu.onion:8333', - 'kswfyurnglm65u7b.onion:8333', - 'ldu2hbiorkvdymja.onion:8333', - 'lftugyhf6vnouikf.onion:8333', - 'ln3csnn6774nzgyn.onion:8333', - 'lvh7k53s62frc6ro.onion:8333', - 'lvvgedppmpigudhz.onion:8333', - 'mbjkotfqcn5gnsrm.onion:8333', - 'mk3bnep5ubou7i44.onion:8333', - 'muhp42ytbwi6qf62.onion:8333', - 'n5khsbd6whw7ooip.onion:8333', - 'na6otugfitr7pnlv.onion:8333', - 'nclrhbeertvin7cu.onion:8333', - 'ndmbrjcvu2s6jcom.onion:8333', - 'nf4iypnyjwfpcjm7.onion:8333', - 'nkdw6ywzt3dqwxuf.onion:8333', - 'nqmxpgrpuysullkq.onion:8333', - 'ntml2aeumyglyjlk.onion:8333', - 'o4sl5na6jeqgi3l6.onion:8333', - 'opencubebqqx3buj.onion:8333', - 'oudab5q7ruclifdv.onion:8333', - 'ovbkvgdllk3xxeah.onion:8333', - 'pg2jeh62fkq3byps.onion:8333', - 'pgufebhe6mt7knqz.onion:8333', - 'pkcgxf23ws3lwqvq.onion:8333', - 'po3j2hfkmf7sh36o.onion:8333', - 'qdtau72ifwauot6b.onion:8333', - 'qidnrqy2ozz3nzqq.onion:8333', - 'qpebweackyztorrm.onion:8333', - 'qsl3x63225alx4bt.onion:8333', - 'readybit5veyche6.onion:8333', - 'rjw6vpw5ffoncxuh.onion:8333', - 's2epxac7ovy36ruj.onion:8333', - 'srkgyv5edn2pa7il.onion:8333', - 'sv5oitfnsmfoc3wu.onion:8333', - 'tdlpmqnpfqehqj7c.onion:8333', - 'ttx7ddwltrixannm.onion:8333', - 'uftbw4zi5wlzcwho.onion:8333', - 'uoailgcebjuws47e.onion:8333', - 'uqvucqhplwqbjrsb.onion:8333', - 'uz3pvdhie3372vxw.onion:8333', - 'v2x7gpj3shxfnl25.onion:8333', - 'vdhrg3k2akmf6kek.onion:8333', - 'vov46htt6gyixdmb.onion:8333', - 'vrfs5jwtfzj2ss6n.onion:8333', - 'vwpcfguewxhky4iy.onion:8333', - 'wg3b3qxcwcrraq2o.onion:8333', - 'wgeecjm4w4ko66f7.onion:8333', - 'wmxc6ask4a5xyaxh.onion:8333', - 'wqrafn4zal3bbbhr.onion:8333', - 'xagzqmjgwgdvl2di.onion:8333', - 'xhi5x5qc44elydk4.onion:8333', - 'xk6bjlmgvwojvozj.onion:8333', - 'xmgr7fsmp7bgburk.onion:8333', - 'xocvz3dzyu2kzu6f.onion:8333', - 'xv7pt6etwxiygss6.onion:8444', - 'xz563swdjd7yqymb.onion:8333', - 'yumx7asj7feoozic.onion:8333', - 'yzmyolvp55rydnsm.onion:8333', - 'z3forfpyjyxxgfr5.onion:8333', - 'z5x2wes6mhbml2t5.onion:8333', - 'zmaddsqelw2oywfb.onion:8444', - 'zqlojwtc4lsurgie.onion:8333', - 'zvwc7ad4m2dvc74x.onion:8333' + // Hosts that serve compact block filters + // dig x49.seed.bitcoin.sipa.be +short + '128.199.197.215', + '72.48.253.168', + '109.202.70.227', + '110.40.210.253', + '223.205.107.43', + '3.123.70.97', + '51.158.164.69', + '95.76.177.15', + '98.127.230.215', + '3.235.63.123', + '50.65.85.9', + '89.58.60.208', + '91.134.145.202', + '142.166.19.23', + '156.146.137.142', + '101.58.39.109', + '5.255.98.79', + '89.203.73.249', + '202.65.65.248', + '188.83.134.205', + '89.102.1.132', + '68.202.132.47', + '88.119.167.62', + '190.64.134.52', + '84.80.197.174', + + // IPv4 hosts + // dig seed.bitcoin.sipa.be +short, + '193.72.32.187', + '154.26.154.73', + '209.188.21.68', + '78.47.61.83', + '76.71.89.120', + '167.86.102.174', + '89.216.91.120', + '185.31.136.246', + '84.80.197.174', + '95.216.33.150', + '100.14.51.167', + '129.13.189.215', + '5.144.84.87', + '91.237.88.218', + '158.140.141.69', + '15.235.54.198', + '18.100.60.149', + '185.8.104.179', + '84.107.90.112', + '65.108.201.169', + '18.117.80.202', + '35.244.108.75', + '98.250.10.88', + '92.105.17.74', + '187.113.127.5', + + // IPv6 hosts + // dig seed.bitcoin.sipa.be AAAA +short', + '2001:470:26:472::b7c', + '2a01:4f8:171:1f16::2', + '2001:470:1b55::', + '2001:470:1b62::', + '2001:470:1f05:43b:2831:8530:7179:5864', + '2001:470:1f15:106:e2d5:5eff:fe42:7ae5', + '2001:470:6c80:3::1', + '2001:569:5079:abd2::c9', + '2001:871:23c:85f4:5a47:caff:fe71:c8d', + '2001:b07:646b:8074:32e8:9243:a337:e60a', + '2a0d:f302:111:bd97::1', + '2001:da8:215:6a01::d648:c48f', + '2001:19f0:5:2b12:5400:4ff:fe6e:3afe', + '2001:4091:a244:841a:4a21:bff:fe51:3d2d' + + // OnionV2 addresses are no longer supported + // by Tor and bcoin does not yet support Onionv3 ]; diff --git a/lib/node/http.js b/lib/node/http.js index 36ea3eb92..90b30134b 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -302,24 +302,6 @@ class HTTP extends Server { res.json(200, filter.toJSON()); }); - this.get('/filterheader/:block', async (req, res) => { - const valid = Validator.fromRequest(req); - const hash = valid.uintbrhash('block'); - - enforce(hash != null, 'Hash or height required.'); - - const filterName = valid.str(1, 'BASIC').toUpperCase(); - const filterHeader = await this.node. - getBlockFilterHeader(hash, filterName); - - if (!filterHeader) { - res.json(404); - return; - } - - res.json(200, filterHeader.toJSON()); - }); - // Mempool snapshot this.get('/mempool', async (req, res) => { enforce(this.mempool, 'No mempool available.'); diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 7e1c4b3ed..31787a443 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -1,5 +1,5 @@ /*! - * neutrino.js - spv node for bcoin + * neutrino.js - neutrino node for bcoin * Copyright (c) 2023, Manav Desai (MIT License) * Copyright (c) 2023, Shaswat Gupta (MIT License). * https://github.com/bcoin-org/bcoin @@ -8,6 +8,7 @@ 'use strict'; const assert = require('bsert'); +const path = require('path'); const Chain = require('../blockchain/chain'); const Pool = require('../net/pool'); const Node = require('./node'); @@ -69,7 +70,7 @@ class Neutrino extends Node { bip91: this.config.bool('bip91'), bip148: this.config.bool('bip148'), spv: true, - neutrino: this.neutrino + location: path.join(this.config.prefix, 'neutrino') }); this.filterIndexers.set( @@ -161,7 +162,7 @@ class Neutrino extends Node { this.emit('reset', tip); }); - this.chain.on('full', async () => { + this.pool.on('headers', async () => { if (this.chain.height === 0) return; this.logger.info('Block Headers are fully synced'); @@ -177,7 +178,7 @@ class Neutrino extends Node { this.pool.on('cfilters', async () => { this.logger.info('Compact Filters are fully synced'); - this.startSync(); + this.pool.forceSync(); }); this.loadPlugins(); @@ -272,16 +273,6 @@ class Neutrino extends Node { return this.broadcast(tx); } - /** - * Broadcast a transaction. Silence errors. - * @param {TX} tx - * @returns {Promise} - */ - - relay(tx) { - return this.broadcast(tx); - } - /** * Connect to the network. * @returns {Promise} diff --git a/lib/node/node.js b/lib/node/node.js index e2e20f453..39c30623f 100644 --- a/lib/node/node.js +++ b/lib/node/node.js @@ -435,28 +435,6 @@ class Node extends EventEmitter { return Indexer.getFilter(hash); } - - /** - * Retrieve compact filter header by hash/height. - * @param {Hash | Number} hash - * @param {Number} type - * @returns {Promise} - Returns {@link Buffer}. - */ - - async getBlockFilterHeader(hash, filterType) { - const Indexer = this.filterIndexers.get(filterType); - - if (!Indexer) - return null; - - if (typeof hash === 'number') - hash = await this.chain.getHash(hash); - - if (!hash) - return null; - - return Indexer.getFilterHeader(hash); - } } /* diff --git a/lib/node/rpc.js b/lib/node/rpc.js index bc1e93d44..ab67affb9 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -155,14 +155,11 @@ class RPC extends RPCBase { this.add('getblockchaininfo', this.getBlockchainInfo); this.add('getbestblockhash', this.getBestBlockHash); this.add('getblockcount', this.getBlockCount); - this.add('getfiltercount', this.getFilterCount); - this.add('getfilterheadercount', this.getFilterHeaderCount); this.add('getblock', this.getBlock); this.add('getblockbyheight', this.getBlockByHeight); this.add('getblockhash', this.getBlockHash); this.add('getblockheader', this.getBlockHeader); this.add('getblockfilter', this.getBlockFilter); - this.add('getblockfilterheader', this.getBlockFilterHeader); this.add('getchaintips', this.getChainTips); this.add('getdifficulty', this.getDifficulty); this.add('getmempoolancestors', this.getMempoolAncestors); @@ -632,22 +629,6 @@ class RPC extends RPCBase { return this.chain.tip.height; } - async getFilterCount(args, help) { - if (help || args.length !== 0) - throw new RPCError(errs.MISC_ERROR, 'getfiltercount'); - - const height = await this.chain.getCFilterHeight(); - return height; - } - - async getFilterHeaderCount(args, help) { - if (help || args.length !== 0) - throw new RPCError(errs.MISC_ERROR, 'getfilterheadercount'); - - const height = await this.chain.getCFHeaderHeight(); - return height; - } - async getBlock(args, help) { if (help || args.length < 1 || args.length > 3) throw new RPCError(errs.MISC_ERROR, 'getblock "hash" ( verbose )'); @@ -788,32 +769,6 @@ class RPC extends RPCBase { return filter.toJSON(); } - async getBlockFilterHeader(args, help) { - if (help || args.length < 1 || args.length > 2) { - throw new RPCError(errs.MISC_ERROR, - 'getblockfilterheader "hash" ( "type" )'); - } - - const valid = new Validator(args); - const hash = valid.brhash(0); - const filterName = valid.str(1, 'BASIC').toUpperCase(); - - const filterType = filters[filterName]; - - if (!hash) - throw new RPCError(errs.MISC_ERROR, 'Invalid block hash.'); - - if (!filterType) - throw new RPCError(errs.MISC_ERROR, 'Filter type not supported'); - - const filterHeader = await this.node.getBlockFilterHeader(hash, filterName); - - if (!filterHeader) - throw new RPCError(errs.MISC_ERROR, 'Block filter header not found.'); - - return filterHeader; - } - async getChainTips(args, help) { if (help || args.length !== 0) throw new RPCError(errs.MISC_ERROR, 'getchaintips'); diff --git a/lib/protocol/networks.js b/lib/protocol/networks.js index 8c2db9e8e..16e6bedf7 100644 --- a/lib/protocol/networks.js +++ b/lib/protocol/networks.js @@ -792,7 +792,7 @@ regtest.block = { bip66hash: null, pruneAfterHeight: 1000, keepBlocks: 10000, - maxTipAge: 24 * 60 * 60, + maxTipAge: 0xffffffff, slowHeight: 0 }; diff --git a/test/neutrino-test.js b/test/neutrino-test.js index 8053fd77b..9e75177f0 100644 --- a/test/neutrino-test.js +++ b/test/neutrino-test.js @@ -43,6 +43,8 @@ describe('Neutrino', function () { memory: true, port: 10000, httpPort: 20000, + logConsole: true, + logLevel: 'debug', neutrino: true, only: '127.0.0.1' }); @@ -65,31 +67,25 @@ describe('Neutrino', function () { it('should get new blocks headers-only', async () => { await mineBlocks(10); - await new Promise(resolve => setTimeout(resolve, 400)); + await forValue(neutrinoNode.chain, 'height', fullNode.chain.height); assert.equal(neutrinoNode.chain.height, fullNode.chain.height); }); - it('should getcfheaders', async () => { - await new Promise(resolve => setTimeout(resolve, 400)); - const headerHeight = await neutrinoNode.chain.getCFHeaderHeight(); - assert.equal(headerHeight, neutrinoNode.chain.height); - }); - - it('should getcfilters', async () => { - await new Promise(resolve => setTimeout(resolve, 400)); - const filterHeight = await neutrinoNode.chain.getCFilterHeight(); + it('should cfheaders and getcfilters', async () => { + const filterIndexer = neutrinoNode.filterIndexers.get('BASIC'); + await forValue(filterIndexer, 'height', neutrinoNode.chain.height); + const filterHeight = filterIndexer.height; assert.equal(filterHeight, neutrinoNode.chain.height); + const headerHeight = await neutrinoNode.pool.cfHeaderChain.tail.height; + assert.equal(headerHeight, neutrinoNode.chain.height); }); it('should save filters correctly', async () => { const filterIndexer = neutrinoNode.filterIndexers.get('BASIC'); for (let i = 0; i < neutrinoNode.chain.height; i++) { const hash = await neutrinoNode.chain.getHash(i); - const filterHeader = await filterIndexer.getFilterHeader(hash); - assert(filterHeader); const filter = await filterIndexer.getFilter(hash); assert(filter); - assert(filterHeader.equals(filter.header)); } }); }); @@ -141,8 +137,26 @@ describe('Neutrino', function () { it('should get new blocks headers-only', async () => { await mineBlocks(10); - await new Promise(resolve => setTimeout(resolve, 400)); + await forValue(neutrinoNode.chain, 'height', fullNode.chain.height); assert.equal(neutrinoNode.chain.height, fullNode.chain.height); }); + + it('should getcfheaders and getcfilters', async () => { + const filterIndexer = neutrinoNode.filterIndexers.get('BASIC'); + await forValue(filterIndexer, 'height', neutrinoNode.chain.height); + const filterHeight = filterIndexer.height; + assert.equal(filterHeight, neutrinoNode.chain.height); + const headerHeight = await neutrinoNode.pool.cfHeaderChain.tail.height; + assert.equal(headerHeight, neutrinoNode.chain.height); + }); + + it('should save filters correctly', async () => { + const filterIndexer = neutrinoNode.filterIndexers.get('BASIC'); + for (let i = 0; i < neutrinoNode.chain.height; i++) { + const hash = await neutrinoNode.chain.getHash(i); + const filter = await filterIndexer.getFilter(hash); + assert(filter); + } + }); }); }); diff --git a/test/node-rpc-test.js b/test/node-rpc-test.js index b44fa0d27..cdef5b4dd 100644 --- a/test/node-rpc-test.js +++ b/test/node-rpc-test.js @@ -189,16 +189,6 @@ describe('RPC', function() { assert.strictEqual(expected.filter, info.filter); }); - it('should rpc getblockfilterheader', async () => { - const hash = await nclient.execute('getblockhash', [node.chain.tip.height]); - const info = await nclient.execute('getblockfilterheader', [hash, 'BASIC']); - const indexer = node.filterIndexers.get('BASIC'); - const filterHeader = await indexer.getFilterHeader(node.chain.tip.hash); - const expected = filterHeader.toJSON(); - - assert.deepStrictEqual(expected, info); - }); - describe('Blockchain', function () { it('should rpc getchaintips', async () => { const info = await nclient.execute('getchaintips', []); diff --git a/test/p2p-bip157-test.js b/test/p2p-bip157-test.js deleted file mode 100644 index b72c7175a..000000000 --- a/test/p2p-bip157-test.js +++ /dev/null @@ -1,100 +0,0 @@ -/* eslint-env mocha */ -/* eslint prefer-arrow-callback: "off" */ - -'use strict'; - -const assert = require('bsert'); -const FullNode = require('../lib/node/fullnode'); -const NeutrinoNode = require('../lib/node/neutrino'); -const {forValue} = require('./util/common'); -const {MAX_CFILTERS} = require('../lib/net/common'); -const packets = require('../lib/net/packets'); - -describe('P2P', function () { - this.timeout(50000); - - const node1 = new NeutrinoNode({ - network: 'regtest', - memory: true, - port: 10000, - httpPort: 20000, - only: '127.0.0.1', - neutrino: true - }); - - const node2 = new FullNode({ - network: 'regtest', - memory: true, - listen: true, - indexFilter: true, - bip157: true - }); - - let peer; - const nodePackets = {}; - - node1.pool.on('packet', (packet) => { - if (!nodePackets[packet.cmd]) - nodePackets[packet.cmd] = [packet]; - else - nodePackets[packet.cmd].push(packet); - }); - - async function mineBlocks(n) { - while (n) { - const block = await node2.miner.mineBlock(); - await node2.chain.add(block); - await new Promise(resolve => setTimeout(resolve, 20)); - n--; - } - await forValue(node1.chain, 'height', node2.chain.height); - } - - before(async () => { - const waitForConnection = new Promise((resolve, reject) => { - node1.pool.once('peer open', async (peer) => { - resolve(peer); - }); - }); - - await node1.open(); - await node2.open(); - await node1.connect(); - await node2.connect(); - node1.startSync(); - node2.startSync(); - - // `peer` is node2, from node1's perspective. - // So peer.send() sends a packet from node1 to node2, - // and `nodePackets` catches the response packets that - // node2 sends back to node1. - peer = await waitForConnection; - }); - - after(async () => { - await node1.close(); - await node2.close(); - }); - - describe('BIP157', function () { - before(async () => { - // Do not exceed limit, including genesis block - await mineBlocks(MAX_CFILTERS - node1.chain.height - 1); - }); - - it('CFCheckpt', async () => { - nodePackets.cfcheckpt = []; - - await mineBlocks(2); - - const pkt = new packets.GetCFCheckptPacket( - 0, - node1.chain.tip.hash - ); - - peer.send(pkt); - await forValue(nodePackets.cfcheckpt, 'length', 1); - assert.strictEqual(nodePackets.cfcheckpt[0].filterHeaders.length, 1); - }); - }); -}); diff --git a/test/p2p-test.js b/test/p2p-test.js index b9502550d..c9c67f35f 100644 --- a/test/p2p-test.js +++ b/test/p2p-test.js @@ -5,79 +5,153 @@ const assert = require('bsert'); const FullNode = require('../lib/node/fullnode'); +const NeutrinoNode = require('../lib/node/neutrino'); const {forValue} = require('./util/common'); +const {MAX_CFILTERS} = require('../lib/net/common'); +const packets = require('../lib/net/packets'); describe('P2P', function () { - this.timeout(5000); - - const node1 = new FullNode({ - network: 'regtest', - memory: true, - port: 10000, - httpPort: 20000, - only: '127.0.0.1' - }); + this.timeout(50000); const node2 = new FullNode({ network: 'regtest', memory: true, listen: true, + logConsole: true, + logLevel: 'debug', indexFilter: true, bip157: true }); let peer; - const nodePackets = {}; - node1.pool.on('packet', (packet) => { - if (!nodePackets[packet.cmd]) - nodePackets[packet.cmd] = [packet]; - else - nodePackets[packet.cmd].push(packet); - }); - - async function mineBlocks(n) { + async function mineBlocks(node, n) { while (n) { const block = await node2.miner.mineBlock(); await node2.chain.add(block); + await new Promise(resolve => setTimeout(resolve, 20)); n--; } - await forValue(node1.chain, 'height', node2.chain.height); + await forValue(node.chain, 'height', node2.chain.height); } - before(async () => { - const waitForConnection = new Promise((resolve, reject) => { - node1.pool.once('peer open', async (peer) => { - resolve(peer); + describe('BIP157', function () { + const node1 = new NeutrinoNode({ + network: 'regtest', + memory: true, + port: 10000, + logConsole: true, + logLevel: 'debug', + httpPort: 20000, + only: '127.0.0.1', + neutrino: true + }); + + const nodePackets = {}; + + node1.pool.on('packet', (packet) => { + if (!nodePackets[packet.cmd]) + nodePackets[packet.cmd] = [packet]; + else + nodePackets[packet.cmd].push(packet); + }); + + before(async () => { + const waitForConnection = new Promise((resolve, reject) => { + node1.pool.once('peer open', async (peer) => { + resolve(peer); + }); }); + + await node1.open(); + await node2.open(); + await node1.connect(); + await node2.connect(); + node1.startSync(); + node2.startSync(); + + // `peer` is node2, from node1's perspective. + // So peer.send() sends a packet from node1 to node2, + // and `nodePackets` catches the response packets that + // node2 sends back to node1. + peer = await waitForConnection; + // Do not exceed limit, including genesis block + await mineBlocks(node1, MAX_CFILTERS - node1.chain.height - 1); }); - await node1.open(); - await node2.open(); - await node1.connect(); - await node2.connect(); - node1.startSync(); - node2.startSync(); - await mineBlocks(1); - - // `peer` is node2, from node1's perspective. - // So peer.send() sends a packet from node1 to node2, - // and `nodePackets` catches the response packets that - // node2 sends back to node1. - peer = await waitForConnection; - }); + after(async () => { + await node1.close(); + await node2.close(); + }); + + it('CFCheckpt', async () => { + nodePackets.cfcheckpt = []; - after(async () => { - await node1.close(); - await node2.close(); + await mineBlocks(node1, 2); + + const pkt = new packets.GetCFCheckptPacket( + 0, + node1.chain.tip.hash + ); + + peer.send(pkt); + await forValue(nodePackets.cfcheckpt, 'length', 1); + assert.strictEqual(nodePackets.cfcheckpt[0].filterHeaders.length, 1); + }); }); describe('Compact Blocks', function () { + const node1 = new FullNode({ + network: 'regtest', + memory: true, + port: 10000, + logConsole: true, + logLevel: 'debug', + httpPort: 20000, + only: '127.0.0.1' + }); + + const nodePackets = {}; + + node1.pool.on('packet', (packet) => { + if (!nodePackets[packet.cmd]) + nodePackets[packet.cmd] = [packet]; + else + nodePackets[packet.cmd].push(packet); + }); + + before(async () => { + const waitForConnection = new Promise((resolve, reject) => { + node1.pool.once('peer open', async (peer) => { + resolve(peer); + }); + }); + + await node1.open(); + await node2.open(); + await node1.connect(); + await node2.connect(); + node1.startSync(); + node2.startSync(); + await mineBlocks(node1, 1); + + // `peer` is node2, from node1's perspective. + // So peer.send() sends a packet from node1 to node2, + // and `nodePackets` catches the response packets that + // node2 sends back to node1. + peer = await waitForConnection; + }); + + after(async () => { + await node1.close(); + await node2.close(); + }); + it('should get compact block in low bandwidth mode', async () => { nodePackets.inv = []; nodePackets.cmpctblock = []; - await mineBlocks(1); + await mineBlocks(node1, 1); assert.strictEqual(nodePackets.inv.length, 1); assert.strictEqual(nodePackets.cmpctblock.length, 1); @@ -90,7 +164,7 @@ describe('P2P', function () { peer.sendCompact(1); node1.pool.options.blockMode = 1; - await mineBlocks(1); + await mineBlocks(node1, 1); assert.strictEqual(nodePackets.inv.length, 0); assert.strictEqual(nodePackets.cmpctblock.length, 1);