From 25518621ad153cd4bea246a4af61abb9f658c2d1 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Mon, 26 Jun 2023 11:40:09 -0500 Subject: [PATCH 1/2] feat: Added wallet side integration for neutrino --- lib/blockchain/chain.js | 42 ++++++++++++++++++++++++++++++++++++++- lib/blockchain/chaindb.js | 14 +++++++++++-- lib/client/node.js | 4 ++++ lib/net/pool.js | 8 ++++++++ lib/node/http.js | 7 +++++++ lib/wallet/client.js | 13 ++++++++++++ lib/wallet/nodeclient.js | 25 ++++++++++++++++++++++- lib/wallet/nullclient.js | 13 ++++++++++++ lib/wallet/wallet.js | 2 ++ lib/wallet/walletdb.js | 39 ++++++++++++++++++++++++++++++++++++ 10 files changed, 163 insertions(+), 4 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 9cd0a312f..2d90f0993 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -64,6 +64,8 @@ class Chain extends AsyncEmitter { this.orphanMap = new BufferMap(); this.orphanPrev = new BufferMap(); + + this.getPrunedMap = new BufferMap(); } /** @@ -1368,7 +1370,18 @@ class Chain extends AsyncEmitter { } // Do we already have this block? - if (await this.hasEntry(hash)) { + const existingEntry = await this.getEntry(hash); + + // FOR EDUCATIONAL PURPOSES ONLY: save block without checking anything + if (existingEntry && this.getPrunedMap.has(hash)) { + block = block.toBlock(); + await this.db.updateNeutrinoSave(); + await this.db.save(existingEntry, block, new CoinView()); + await this.db.updateNeutrinoSave(); + return existingEntry; + } + + if (existingEntry) { this.logger.debug('Already have block: %h.', block.hash()); throw new VerifyError(block, 'duplicate', 'duplicate', 0); } @@ -1925,6 +1938,33 @@ class Chain extends AsyncEmitter { return this.db.getBlock(hash); } + async getBlockPeer(hash, filter) { + let block = await this.db.getBlock(hash); + if (block) { + let entry = await this.getEntry(hash); + assert(entry.hash.equals(hash)); + return block; + } else { + this.logger.warning('Block not found, attempting to download'); + + // Ensure hash not height + hash = await this.db.getHash(hash); + + // FOR EDUCATIONAL PURPOSES ONLY: flag block for re-downloading + const wait = new Promise((resolve, reject) => { + this.getPrunedMap.set(hash, resolve); + }); + + await this.emitAsync('getprunedblock', hash); + await wait; + block = await this.db.getBlock(hash); + let entry = await this.getEntry(hash); + assert(entry.hash.equals(hash)); + + return block; + } + } + /** * Retrieve a block from the database (not filled with coins). * @param {Hash} block diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index cb91accaa..d638f7be3 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -47,6 +47,8 @@ class ChainDB { this.pending = null; this.current = null; + this.neutrinoSave = false; + this.cacheHash = new LRU(this.options.entryCache, null, BufferMap); this.cacheHeight = new LRU(this.options.entryCache); @@ -1001,7 +1003,7 @@ class ChainDB { */ async getRawBlock(block) { - if (this.options.spv) + if (this.options.spv && !this.options.neutrino) return null; const hash = await this.getHash(block); @@ -1150,6 +1152,14 @@ class ChainDB { * @returns {Promise} */ + async updateNeutrinoSave () { + if(this.neutrinoSave) { + this.neutrinoSave = false; + } else { + this.neutrinoSave = true; + } + } + async save(entry, block, view) { this.start(); try { @@ -1478,7 +1488,7 @@ class ChainDB { async saveBlock(entry, block, view) { const hash = block.hash(); - if (this.options.spv) + if (this.options.spv && !this.neutrinoSave) return; // Write actual block data. diff --git a/lib/client/node.js b/lib/client/node.js index 50800cac1..7822408d1 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -169,6 +169,10 @@ class NodeClient extends Client { return this.get(`/filter/${filter}`); } + checkKeyFilter(key, filter) { + return this.call('check key filter', key, filter); + } + /** * Add a transaction to the mempool and broadcast it. * @param {TX} tx diff --git a/lib/net/pool.js b/lib/net/pool.js index 234b23bc2..8fd45e7c6 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -2316,6 +2316,14 @@ class Pool extends EventEmitter { } // Block was orphaned. + + const resolve = this.chain.getPrunedMap.get(hash); + if (resolve) { + this.logger.warning('Received pruned block by special request'); + this.chain.getPrunedMap.delete(hash); + resolve(); + } + if (!entry) { if (this.checkpoints) { this.logger.warning( diff --git a/lib/node/http.js b/lib/node/http.js index 8448ec015..838aa9be7 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -498,6 +498,13 @@ class HTTP extends Server { return null; }); + socket.hook('check key filter', (...args) => { + const valid = new Validator(args); + const key = valid.buf(0); + const filter = valid.buf(1); + return this.pool.checkKeyFilter(key, filter); + }); + socket.hook('estimate fee', (...args) => { const valid = new Validator(args); const blocks = valid.u32(0); diff --git a/lib/wallet/client.js b/lib/wallet/client.js index 768f38e50..93c521581 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -12,6 +12,8 @@ const NodeClient = require('../client/node'); const util = require('../utils/util'); const TX = require('../primitives/tx'); const hash256 = require('bcrypto/lib/hash256'); +const WalletKey = require('./walletkey'); +const Filter = require('../primitives/filter'); const parsers = { 'block connect': (entry, txs) => parseBlock(entry, txs), @@ -71,6 +73,17 @@ class WalletClient extends NodeClient { return super.setFilter(filter.toRaw()); } + /** + * Check filter against wallet key ring + * @param {WalletKey} ring + * @param {Filter} filter + * @returns {Promise} + */ + + async checkKeyFilter(ring, filter) { + return super.checkKeyFilter(ring, filter); + } + async rescan(start) { if (Buffer.isBuffer(start)) start = util.revHex(start); diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 9f6c43600..8fc9d1c0b 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -8,6 +8,8 @@ const assert = require('bsert'); const AsyncEmitter = require('bevent'); +const WalletKey = require('./walletkey'); +const Filter = require('../primitives/filter'); /** * Node Client @@ -37,7 +39,7 @@ class NodeClient extends AsyncEmitter { init() { this.node.chain.on('connect', async (entry, block) => { - if (!this.opened) + if (!this.opened || this.node.neutrino) return; await this.emitAsync('block connect', entry, block.txs); @@ -50,6 +52,12 @@ class NodeClient extends AsyncEmitter { await this.emitAsync('block disconnect', entry); }); + this.node.pool.on('cfilter', async (blockHeight, filter) => { + if(!this.opened) return; + + await this.emitAsync('cfilter', blockHeight, filter); + }) + this.node.on('tx', (tx) => { if (!this.opened) return; @@ -134,6 +142,10 @@ class NodeClient extends AsyncEmitter { return entry; } + async getBlockFromNode(hash, filter) { + await this.node.chain.getBlockPeer(hash, filter); + } + /** * Send a transaction. Do not wait for promise. * @param {TX} tx @@ -174,6 +186,17 @@ class NodeClient extends AsyncEmitter { this.node.pool.queueFilterLoad(); } + /** + * Check filter against wallet key ring + * @param {WalletKey} ring + * @param {Filter} filter + * @returns {Promise} + */ + + async checkKeyFilter(ring, filter) { + this.node.pool.checkKeyFilter(ring, filter); + } + /** * Estimate smart fee. * @param {Number?} blocks diff --git a/lib/wallet/nullclient.js b/lib/wallet/nullclient.js index 744629d4b..68221a397 100644 --- a/lib/wallet/nullclient.js +++ b/lib/wallet/nullclient.js @@ -8,6 +8,8 @@ const assert = require('bsert'); const EventEmitter = require('events'); +const WalletKey = require('./walletkey'); +const Filter = require('../primitives/filter'); /** * Null Client @@ -130,6 +132,17 @@ class NullClient extends EventEmitter { this.wdb.emit('reset filter'); } + /** + * Check filter against wallet key ring + * @param {WalletKey} ring + * @param {Filter} filter + * @returns {Promise} + */ + + async checkKeyFilter(ring, filter) { + ; + } + /** * Esimate smart fee. * @param {Number?} blocks diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 3703deaf9..32bdb4b38 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -59,6 +59,8 @@ class Wallet extends EventEmitter { this.writeLock = new Lock(); this.fundLock = new Lock(); + this.neutrino = false; + this.wid = 0; this.id = null; this.watchOnly = false; diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index b1f57ebf0..2f179644e 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -169,6 +169,14 @@ class WalletDB extends EventEmitter { this.emit('error', e); } }); + + this.client.bind('cfilter', async (blockHeight, filter) => { + try { + await this.checkFilter(blockHeight, filter); + } catch (e) { + this.emit('error', e); + } + }) } /** @@ -568,6 +576,37 @@ class WalletDB extends EventEmitter { return this.client.resetFilter(); } + async checkFilter (blockHash, filter) { + const gcsKey = blockHash.slice(0, 16); + + const piter = this.db.iterator({ + gte: layout.p.min(), + lte: layout.p.max() + }); + + await piter.each(async (key) => { + const [data] = layout.p.decode(key); + // todo: check filter + let match = filter.match(gcsKey, data); + if (match) + await this.client.getBlockFromNode(blockHash, filter); + }); + + const oiter = this.db.iterator({ + gte: layout.o.min(), + lte: layout.o.max() + }); + + await oiter.each(async (key) => { + const [hash, index] = layout.o.decode(key); + const outpoint = new Outpoint(hash, index); + const data = outpoint.toRaw(); + let match = filter.match(gcsKey, data); + if (match) + await this.client.getBlockFromNode(blockHash, filter); + }); + } + /** * Backup the wallet db. * @param {String} path From d07744f36ae7e684b1820299c2f83e855ec6e6be Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Mon, 26 Jun 2023 12:46:20 -0500 Subject: [PATCH 2/2] feat: wallet integration complete --- lib/client/node.js | 4 ++-- lib/node/http.js | 8 ++++---- lib/wallet/client.js | 4 ++-- lib/wallet/nodeclient.js | 4 ---- lib/wallet/nullclient.js | 2 +- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/client/node.js b/lib/client/node.js index 7822408d1..96b5ac81f 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -169,8 +169,8 @@ class NodeClient extends Client { return this.get(`/filter/${filter}`); } - checkKeyFilter(key, filter) { - return this.call('check key filter', key, filter); + getBlockPeer(hash, filter) { + return this.call('get block peer', hash, filter); } /** diff --git a/lib/node/http.js b/lib/node/http.js index 838aa9be7..1dfb2ddb2 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -498,12 +498,12 @@ class HTTP extends Server { return null; }); - socket.hook('check key filter', (...args) => { + socket.hook('get block peer', (...args) => { const valid = new Validator(args); - const key = valid.buf(0); + const hash = valid.hash(0); const filter = valid.buf(1); - return this.pool.checkKeyFilter(key, filter); - }); + return this.pool.getBlockPeer(hash, filter); + }) socket.hook('estimate fee', (...args) => { const valid = new Validator(args); diff --git a/lib/wallet/client.js b/lib/wallet/client.js index 93c521581..ae8681b6c 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -80,8 +80,8 @@ class WalletClient extends NodeClient { * @returns {Promise} */ - async checkKeyFilter(ring, filter) { - return super.checkKeyFilter(ring, filter); + async getBlockFromNode(hash, filter) { + return super.getBlockPeer(hash, filter); } async rescan(start) { diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 8fc9d1c0b..2b71ba61f 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -193,10 +193,6 @@ class NodeClient extends AsyncEmitter { * @returns {Promise} */ - async checkKeyFilter(ring, filter) { - this.node.pool.checkKeyFilter(ring, filter); - } - /** * Estimate smart fee. * @param {Number?} blocks diff --git a/lib/wallet/nullclient.js b/lib/wallet/nullclient.js index 68221a397..7a683b5ca 100644 --- a/lib/wallet/nullclient.js +++ b/lib/wallet/nullclient.js @@ -139,7 +139,7 @@ class NullClient extends EventEmitter { * @returns {Promise} */ - async checkKeyFilter(ring, filter) { + async getBlockFromNode(hash, filter) { ; }