diff --git a/CHANGELOG.md b/CHANGELOG.md index 96a52e3b1..fa7b70afe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ process and allows parallel rescans. - `compactInterval` - what is the current compaction interval config. - `nextCompaction` - when will the next compaction trigger after restart. - `lastCompaction` - when was the last compaction run. - - Introduce `scan interactive` hook (start, filter) + - Introduce `scan interactive` hook (start, filter, fullLock) ### Node HTTP Client: - Introduce `scanInteractive` method that starts interactive rescan. @@ -43,6 +43,9 @@ process and allows parallel rescans. - Add `getFee`, an HTTP alternative to estimateFee socket call. ### Wallet Changes +- Add migration that recalculates txdb balances to fix any inconsistencies. +- Wallet will now use `interactive scan` for initial sync(on open) and rescan. + #### Configuration - Wallet now has option `wallet-migrate-no-rescan`/`migrate-no-rescan` if you want to disable rescan when migration recommends it. It may result in the @@ -54,7 +57,7 @@ process and allows parallel rescans. #### Wallet API -- Add migration that recalculates txdb balances to fix any inconsistencies. +- WalletNode now emits `open` and `close` events. - WalletDB Now emits events for: `open`, `close`, `connect`, `disconnect`. - WalletDB - `open()` no longer calls `connect` and needs separate call `connect`. @@ -62,13 +65,14 @@ process and allows parallel rescans. sync to do the rescan. - emits events for: `open`, `close`, `connect`, `disconnect`, `sync done`. -### Wallet HTTP Client +#### Wallet HTTP - All transaction creating endpoints now accept `hardFee` for specifying the exact fee. - All transaction sending endpoints now fundlock/queue tx creation. (no more conflicting transactions) - Add options to `getNames` for passing `own`. + ## v6.0.0 ### Node and Wallet HTTP API diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index aa5bb4c5a..3389ce02d 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2266,16 +2266,55 @@ class Chain extends AsyncEmitter { } } + /** @typedef {import('./common').ScanAction} ScanAction */ + + /** + * @callback ScanInteractiveIterCB + * @param {ChainEntry} entry + * @param {TX[]} txs + * @returns {Promise} + */ + /** * Interactive scan the blockchain for transactions containing specified * address hashes. Allows repeat and abort. * @param {Hash|Number} start - Block hash or height to start at. * @param {BloomFilter} filter - Starting bloom filter containing tx, * address and name hashes. - * @param {Function} iter - Iterator. + * @param {ScanInteractiveIterCB} iter - Iterator. + * @param {Boolean} [fullLock=false] + * @returns {Promise} */ - async scanInteractive(start, filter, iter) { + async scanInteractive(start, filter, iter, fullLock = false) { + if (fullLock) { + const unlock = await this.locker.lock(); + try { + // We lock the whole chain, no longer lock per block scan. + return await this._scanInteractive(start, filter, iter, false); + } catch (e) { + this.logger.debug('Scan(interactive) errored. Error: %s', e.message); + throw e; + } finally { + unlock(); + } + } + + return this._scanInteractive(start, filter, iter, true); + } + + /** + * Interactive scan the blockchain for transactions containing specified + * address hashes. Allows repeat and abort. + * @param {Hash|Number} start - Block hash or height to start at. + * @param {BloomFilter} filter - Starting bloom filter containing tx, + * address and name hashes. + * @param {ScanInteractiveIterCB} iter - Iterator. + * @param {Boolean} [lockPerScan=true] - if we should lock per block scan. + * @returns {Promise} + */ + + async _scanInteractive(start, filter, iter, lockPerScan = true) { if (start == null) start = this.network.genesis.hash; @@ -2287,7 +2326,10 @@ class Chain extends AsyncEmitter { let hash = start; while (hash != null) { - const unlock = await this.locker.lock(); + let unlock; + + if (lockPerScan) + unlock = await this.locker.lock(); try { const {entry, txs} = await this.db.scanBlock(hash, filter); @@ -2333,7 +2375,8 @@ class Chain extends AsyncEmitter { this.logger.debug('Scan(interactive) errored. Error: %s', e.message); throw e; } finally { - unlock(); + if (lockPerScan) + unlock(); } } } diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 7f075349b..1349c0ea7 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -1612,12 +1612,18 @@ class ChainDB { this.logger.info('Finished scanning %d blocks.', total); } + /** + * @typedef {Object} ScanBlockResult + * @property {ChainEntry} entry + * @property {TX[]} txs + */ + /** * Interactive scans block checks. * @param {Hash|Number} blockID - Block hash or height to start at. * @param {BloomFilter} [filter] - Starting bloom filter containing tx, * address and name hashes. - * @returns {Promise} + * @returns {Promise} */ async scanBlock(blockID, filter) { diff --git a/lib/blockchain/common.js b/lib/blockchain/common.js index 2a45e2d45..6a9ab8339 100644 --- a/lib/blockchain/common.js +++ b/lib/blockchain/common.js @@ -84,3 +84,39 @@ exports.scanActions = { REPEAT_ADD: 4, REPEAT: 5 }; + +/** + * @typedef {Object} ActionAbort + * @property {exports.scanActions} type - ABORT + */ + +/** + * @typedef {Object} ActionNext + * @property {exports.scanActions} type - NEXT + */ + +/** + * @typedef {Object} ActionRepeat + * @property {exports.ScanAction} type - REPEAT + */ + +/** + * @typedef {Object} ActionRepeatAdd + * @property {exports.scanActions} type - REPEAT_ADD + * @property {Buffer[]} chunks + */ + +/** + * @typedef {Object} ActionRepeatSet + * @property {exports.scanActions} type - REPEAT_SET + * @property {BloomFilter} filter + */ + +/** + * @typedef {ActionAbort + * | ActionNext + * | ActionRepeat + * | ActionRepeatAdd + * | ActionRepeatSet + * } ScanAction + */ diff --git a/lib/client/node.js b/lib/client/node.js index 631eb3d02..aae765d5c 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -370,16 +370,17 @@ class NodeClient extends Client { * Rescan for any missed transactions. (Interactive) * @param {Number|Hash} start - Start block. * @param {BloomFilter} [filter] + * @param {Boolean} [fullLock=false] * @returns {Promise} */ - rescanInteractive(start, filter = null) { + rescanInteractive(start, filter = null, fullLock = false) { if (start == null) start = 0; assert(typeof start === 'number' || Buffer.isBuffer(start)); - return this.call('rescan interactive', start, filter); + return this.call('rescan interactive', start, filter, fullLock); } } diff --git a/lib/client/wallet.js b/lib/client/wallet.js index 1a027ce45..784b21d17 100644 --- a/lib/client/wallet.js +++ b/lib/client/wallet.js @@ -1600,4 +1600,6 @@ class Wallet extends EventEmitter { * Expose */ +WalletClient.Wallet = Wallet; + module.exports = WalletClient; diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index de7fcc2b0..0e8bbdcb8 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -369,11 +369,13 @@ class FullNode extends Node { * @param {Number|Hash} start - Start block. * @param {BloomFilter} filter * @param {Function} iter - Iterator. + * @param {Boolean} [fullLock=false] - lock the whole chain instead of per + * scan. * @returns {Promise} */ - scanInteractive(start, filter, iter) { - return this.chain.scanInteractive(start, filter, iter); + scanInteractive(start, filter, iter, fullLock = false) { + return this.chain.scanInteractive(start, filter, iter, fullLock); } /** diff --git a/lib/node/http.js b/lib/node/http.js index e82b482bd..c30db1ff1 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -712,6 +712,7 @@ class HTTP extends Server { const valid = new Validator(args); const start = valid.uintbhash(0); const rawFilter = valid.buf(1); + const fullLock = valid.bool(2, false); let filter = socket.filter; if (start == null) @@ -720,7 +721,7 @@ class HTTP extends Server { if (rawFilter) filter = BloomFilter.fromRaw(rawFilter); - return this.scanInteractive(socket, start, filter); + return this.scanInteractive(socket, start, filter, fullLock); }); } @@ -859,10 +860,11 @@ class HTTP extends Server { * @param {WebSocket} socket * @param {Hash} start * @param {BloomFilter} filter + * @param {Boolean} [fullLock=false] * @returns {Promise} */ - async scanInteractive(socket, start, filter) { + async scanInteractive(socket, start, filter, fullLock = false) { const iter = async (entry, txs) => { const block = entry.encode(); const raw = []; @@ -921,12 +923,11 @@ class HTTP extends Server { }; try { - await this.node.scanInteractive(start, filter, iter); + await this.node.scanInteractive(start, filter, iter, fullLock); } catch (err) { - return socket.call('block rescan interactive abort', err.message); + await socket.call('block rescan interactive abort', err.message); + throw err; } - - return null; } } diff --git a/lib/wallet/account.js b/lib/wallet/account.js index d60c7903b..582d3a747 100644 --- a/lib/wallet/account.js +++ b/lib/wallet/account.js @@ -512,7 +512,7 @@ class Account extends bio.Struct { * Allocate new lookahead addresses if necessary. * @param {Number} receiveDepth * @param {Number} changeDepth - * @returns {Promise} - Returns {@link WalletKey}. + * @returns {Promise} */ async syncDepth(b, receive, change) { diff --git a/lib/wallet/client.js b/lib/wallet/client.js index 92bb9a9f8..0f7e3a729 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -11,11 +11,13 @@ const NodeClient = require('../client/node'); const TX = require('../primitives/tx'); const Coin = require('../primitives/coin'); const NameState = require('../covenants/namestate'); +const {encoding} = require('bufio'); const parsers = { 'block connect': (entry, txs) => parseBlock(entry, txs), 'block disconnect': entry => [parseEntry(entry)], 'block rescan': (entry, txs) => parseBlock(entry, txs), + 'block rescan interactive': (entry, txs) => parseBlock(entry, txs), 'chain reset': entry => [parseEntry(entry)], 'tx': tx => [TX.decode(tx)] }; @@ -74,10 +76,27 @@ class WalletClient extends NodeClient { return super.setFilter(filter.encode()); } + /** + * Rescan for any missed transactions. + * @param {Number|Hash} start - Start block. + * @returns {Promise} + */ + async rescan(start) { return super.rescan(start); } + /** + * Rescan interactive for any missed transactions. + * @param {Number|Hash} start - Start block. + * @param {Boolean} [fullLock=false] + * @returns {Promise} + */ + + async rescanInteractive(start, fullLock) { + return super.rescanInteractive(start, null, fullLock); + } + async getNameStatus(nameHash) { const json = await super.getNameStatus(nameHash); return NameState.fromJSON(json); @@ -94,6 +113,9 @@ class WalletClient extends NodeClient { */ function parseEntry(data) { + if (!data) + return null; + // 32 hash // 4 height // 4 nonce @@ -112,17 +134,24 @@ function parseEntry(data) { assert(Buffer.isBuffer(data)); // Just enough to read the three data below - assert(data.length >= 44); + assert(data.length >= 80); + + const hash = data.slice(0, 32); + const height = encoding.readU32(data, 32); + const time = encoding.readU64(data, 40); + const prevBlock = data.slice(48, 80); return { - hash: data.slice(0, 32), - height: data.readUInt32LE(32), - time: data.readUInt32LE(40) + hash, + height, + time, + prevBlock }; } function parseBlock(entry, txs) { const block = parseEntry(entry); + assert(block); const out = []; for (const tx of txs) diff --git a/lib/wallet/node.js b/lib/wallet/node.js index 53707c661..ff4816320 100644 --- a/lib/wallet/node.js +++ b/lib/wallet/node.js @@ -113,6 +113,7 @@ class WalletNode extends Node { await this.handleOpen(); this.logger.info('Wallet node is loaded.'); + this.emit('open'); } /** @@ -134,6 +135,7 @@ class WalletNode extends Node { await this.wdb.disconnect(); await this.wdb.close(); await this.handleClose(); + this.emit('close'); } } diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 40594d03e..5f181bcbc 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -7,6 +7,7 @@ 'use strict'; const assert = require('bsert'); +const blacklist = require('bsock/lib/blacklist'); const AsyncEmitter = require('bevent'); /** @@ -27,6 +28,7 @@ class NodeClient extends AsyncEmitter { this.network = node.network; this.filter = null; this.opened = false; + this.hooks = new Map(); this.init(); } @@ -98,13 +100,46 @@ class NodeClient extends AsyncEmitter { } /** - * Add a listener. - * @param {String} type + * Add a hook. + * @param {String} event * @param {Function} handler */ - hook(type, handler) { - return this.on(type, handler); + hook(event, handler) { + assert(typeof event === 'string', 'Event must be a string.'); + assert(typeof handler === 'function', 'Handler must be a function.'); + assert(!this.hooks.has(event), 'Hook already bound.'); + assert(!Object.prototype.hasOwnProperty.call(blacklist, event), + 'Blacklisted event.'); + this.hooks.set(event, handler); + } + + /** + * Remove a hook. + * @param {String} event + */ + + unhook(event) { + assert(typeof event === 'string', 'Event must be a string.'); + assert(!Object.prototype.hasOwnProperty.call(blacklist, event), + 'Blacklisted event.'); + this.hooks.delete(event); + } + + /** + * Call a hook. + * @param {String} event + * @param {...Object} args + * @returns {Promise} + */ + + handleCall(event, ...args) { + const hook = this.hooks.get(event); + + if (!hook) + throw new Error('No hook available.'); + + return hook(...args); } /** @@ -215,8 +250,6 @@ class NodeClient extends AsyncEmitter { /** * Rescan for any missed transactions. * @param {Number|Hash} start - Start block. - * @param {Bloom} filter - * @param {Function} iter - Iterator. * @returns {Promise} */ @@ -225,10 +258,38 @@ class NodeClient extends AsyncEmitter { return this.node.chain.reset(start); return this.node.chain.scan(start, this.filter, (entry, txs) => { - return this.emitAsync('block rescan', entry, txs); + return this.handleCall('block rescan', entry, txs); }); } + /** + * Rescan interactive for any missed transactions. + * @param {Number|Hash} start - Start block. + * @param {Boolean} [fullLock=false] + * @returns {Promise} + */ + + async rescanInteractive(start, fullLock = true) { + if (this.node.spv) + return this.node.chain.reset(start); + + const iter = async (entry, txs) => { + return await this.handleCall('block rescan interactive', entry, txs); + }; + + try { + return await this.node.scanInteractive( + start, + this.filter, + iter, + fullLock + ); + } catch (e) { + await this.handleCall('block rescan interactive abort', e.message); + throw e; + } + } + /** * Get name state. * @param {Buffer} nameHash diff --git a/lib/wallet/nullclient.js b/lib/wallet/nullclient.js index abc30f266..0dcdde0b7 100644 --- a/lib/wallet/nullclient.js +++ b/lib/wallet/nullclient.js @@ -165,8 +165,6 @@ class NullClient extends EventEmitter { /** * Rescan for any missed transactions. * @param {Number|Hash} start - Start block. - * @param {Bloom} filter - * @param {Function} iter - Iterator. * @returns {Promise} */ @@ -174,6 +172,17 @@ class NullClient extends EventEmitter { ; } + /** + * Rescan interactive for any missed transactions. + * @param {Number|Hash} start - Start block. + * @param {Boolean} [fullLock=false] + * @returns {Promise} + */ + + async rescanInteractive(start, fullLock) { + ; + } + /** * Get opening bid height. * @param {Buffer} nameHash diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index f92ef8103..2d92e6bd1 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -14,14 +14,13 @@ const Amount = require('../ui/amount'); const CoinView = require('../coins/coinview'); const Coin = require('../primitives/coin'); const Outpoint = require('../primitives/outpoint'); -const records = require('./records'); const layout = require('./layout').txdb; const consensus = require('../protocol/consensus'); const policy = require('../protocol/policy'); const rules = require('../covenants/rules'); const NameState = require('../covenants/namestate'); const NameUndo = require('../covenants/undo'); -const {TXRecord} = records; +const {TXRecord} = require('./records'); const {types} = rules; /* @@ -1486,7 +1485,7 @@ class TXDB { /** * Revert a block. * @param {Number} height - * @returns {Promise} - blocks length + * @returns {Promise} - number of txs removed. */ async revert(height) { diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 1d5a8ce79..0423eeb4f 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -46,6 +46,12 @@ const Outpoint = require('../primitives/outpoint'); const EMPTY = Buffer.alloc(0); +/** + * @typedef {Object} AddResult + * @property {Details} details + * @property {WalletKey[]} derived + */ + /** * Wallet * @alias module:wallet.Wallet @@ -4360,7 +4366,7 @@ class Wallet extends EventEmitter { * This is used for deriving new addresses when * a confirmed transaction is seen. * @param {TX} tx - * @returns {Promise} + * @returns {Promise} - derived rings. */ async syncOutputDepth(tx) { @@ -4737,7 +4743,7 @@ class Wallet extends EventEmitter { /** * Add a transaction to the wallets TX history. * @param {TX} tx - * @returns {Promise} + * @returns {Promise} */ async add(tx, block) { @@ -4754,27 +4760,32 @@ class Wallet extends EventEmitter { * Potentially resolves orphans. * @private * @param {TX} tx - * @returns {Promise} + * @returns {Promise} */ async _add(tx, block) { const details = await this.txdb.add(tx, block); - if (details) { - const derived = await this.syncOutputDepth(tx); - if (derived.length > 0) { - this.wdb.emit('address', this, derived); - this.emit('address', derived); - } + if (!details) + return null; + + const derived = await this.syncOutputDepth(tx); + + if (derived.length > 0) { + this.wdb.emit('address', this, derived); + this.emit('address', derived); } - return details; + return { + details, + derived + }; } /** * Revert a block. * @param {Number} height - * @returns {Promise} + * @returns {Promise} - number of txs removed. */ async revert(height) { diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 1e0568007..202f50038 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -30,6 +30,10 @@ const layout = layouts.wdb; const tlayout = layouts.txdb; const {states} = require('../covenants/namestate'); const util = require('../utils/util'); +const {scanActions} = require('../blockchain/common'); + +/** @typedef {import('../primitives/tx')} TX */ +/** @typedef {import('../blockchain/common').ScanAction} ScanAction */ const { ChainState, @@ -38,6 +42,17 @@ const { MapRecord } = records; +/** + * @typedef {Object} AddBlockResult + * @property {Number} txs - Number of transactions added on this add. + * @property {Boolean} filterUpdated - Whether the bloom filter was updated. + */ + +/** + * @typedef {Object} AddTXResult + * @property {Number} wids - Wallet IDs affected. + * @property {Boolean} filterUpdated - Whether the bloom filter was updated. + /** * WalletDB * @alias module:wallet.WalletDB @@ -159,6 +174,9 @@ class WalletDB extends EventEmitter { }); this.client.bind('block disconnect', async (entry) => { + if (this.rescanning) + return; + try { await this.removeBlock(entry); } catch (e) { @@ -174,6 +192,21 @@ class WalletDB extends EventEmitter { } }); + this.client.hook('block rescan interactive', async (entry, txs) => { + try { + return await this.rescanBlockInteractive(entry, txs); + } catch (e) { + this.emit('error', e); + return { + type: scanActions.ABORT + }; + } + }); + + this.client.hook('block rescan interactive abort', async (message) => { + this.emit('error', new Error(message)); + }); + this.client.bind('tx', async (tx) => { try { await this.addTX(tx); @@ -507,23 +540,29 @@ class WalletDB extends EventEmitter { } // syncNode sets the rescanning to true. - return this.scan(height); + return this.scanInteractive(height); } /** * Rescan blockchain from a given height. * Needs this.rescanning = true to be set from the caller. * @private - * @param {Number?} height + * @param {Number} [height=this.state.startHeight] * @returns {Promise} */ async scan(height) { + assert(this.rescanning, 'WDB: Rescanning guard not set.'); + if (height == null) height = this.state.startHeight; assert((height >>> 0) === height, 'WDB: Must pass in a height.'); + this.logger.info( + 'Rolling back %d blocks.', + this.height - height + 1); + await this.rollback(height); this.logger.info( @@ -535,6 +574,38 @@ class WalletDB extends EventEmitter { return this.client.rescan(tip.hash); } + /** + * Interactive scan blockchain from a given height. + * Expect this.rescanning to be set to true. + * @private + * @param {Number} [height=this.state.startHeight] + * @param {Boolean} [fullLock=true] + * @returns {Promise} + */ + + async scanInteractive(height, fullLock = true) { + assert(this.rescanning, 'WDB: Rescanning guard not set.'); + + if (height == null) + height = this.state.startHeight; + + assert((height >>> 0) === height, 'WDB: Must pass in a height.'); + + this.logger.info( + 'Rolling back %d blocks.', + this.height - height + 1); + + await this.rollback(height); + + this.logger.info( + 'WalletDB is scanning %d blocks.', + this.state.height - height + 1); + + const tip = await this.getTip(); + + return this.client.rescanInteractive(tip.hash, fullLock); + } + /** * Deep Clean: * Keep all keys, account data, wallet maps (name and path). @@ -645,7 +716,7 @@ class WalletDB extends EventEmitter { this.rescanning = true; try { - return await this.scan(height); + return await this.scanInteractive(height); } finally { this.rescanning = false; } @@ -2117,7 +2188,7 @@ class WalletDB extends EventEmitter { */ async addOutpointMap(b, hash, index, wid) { - await this.addOutpoint(hash, index); + this.addOutpoint(hash, index); return this.addMap(b, layout.o.encode(hash, index), wid); } @@ -2178,7 +2249,7 @@ class WalletDB extends EventEmitter { /** * Get a wallet block meta. - * @param {Hash} hash + * @param {Number} height * @returns {Promise} */ @@ -2284,7 +2355,8 @@ class WalletDB extends EventEmitter { /** * Add a block's transactions and write the new best hash. * @param {ChainEntry} entry - * @returns {Promise} + * @param {TX[]} txs + * @returns {Promise} */ async addBlock(entry, txs) { @@ -2302,7 +2374,7 @@ class WalletDB extends EventEmitter { * @private * @param {ChainEntry} entry * @param {TX[]} txs - * @returns {Promise} + * @returns {Promise} */ async _addBlock(entry, txs) { @@ -2312,7 +2384,18 @@ class WalletDB extends EventEmitter { this.logger.warning( 'WalletDB is connecting low blocks (%d).', tip.height); - return 0; + + const block = await this.getBlock(tip.height); + assert(block); + + if (!entry.hash.equals(block.hash)) { + // Maybe we run syncChain here. + this.logger.warning( + 'Unusual reorg at low height (%d).', + tip.height); + } + + return null; } if (tip.height >= this.network.block.slowHeight) @@ -2326,12 +2409,41 @@ class WalletDB extends EventEmitter { // updated before the block was fully // processed (in the case of a crash). this.logger.warning('Already saw WalletDB block (%d).', tip.height); + + const block = await this.getBlock(tip.height); + assert(block); + + if (!entry.hash.equals(block.hash)) { + this.logger.warning( + 'Unusual reorg at the same height (%d).', + tip.height); + + // Maybe we can run syncChain here. + return null; + } } else if (tip.height !== this.state.height + 1) { await this._rescan(this.state.height); - return 0; + return null; + } + + let block; + + if (tip.height > 2) { + block = await this.getBlock(tip.height - 1); + assert(block); + } + + if (block && !block.hash.equals(entry.prevBlock)) { + // We can trigger syncChain here as well. + this.logger.warning( + 'Unusual reorg at height (%d).', + tip.height); + + return null; } const walletTxs = []; + let filterUpdated = false; try { // We set the state as confirming so that @@ -2341,8 +2453,13 @@ class WalletDB extends EventEmitter { this.confirming = true; for (const tx of txs) { - if (await this._addTX(tx, tip)) { + const txadded = await this._addTX(tx, tip); + + if (txadded) { walletTxs.push(tx); + + if (txadded.filterUpdated) + filterUpdated = true; } } @@ -2360,14 +2477,17 @@ class WalletDB extends EventEmitter { this.emit('block connect', entry, walletTxs); - return walletTxs.length; + return { + txs: walletTxs.length, + filterUpdated: filterUpdated + }; } /** * Unconfirm a block's transactions * and write the new best hash (SPV version). * @param {ChainEntry} entry - * @returns {Promise} + * @returns {Promise} - number of txs removed. */ async removeBlock(entry) { @@ -2383,7 +2503,7 @@ class WalletDB extends EventEmitter { * Unconfirm a block's transactions. * @private * @param {ChainEntry} entry - * @returns {Promise} + * @returns {Promise} - number of txs removed. */ async _removeBlock(entry) { @@ -2460,12 +2580,47 @@ class WalletDB extends EventEmitter { } } + /** + * Rescan a block interactively. + * @param {ChainEntry} entry + * @param {TX[]} txs + * @returns {Promise} - interactive action + */ + + async rescanBlockInteractive(entry, txs) { + if (!this.rescanning) + throw new Error(`WDB: Unsolicited rescan block: ${entry.height}.`); + + if (entry.height > this.state.height + 1) + throw new Error(`WDB: Rescan block too high: ${entry.height}.`); + + const blockAdded = await this._addBlock(entry, txs); + + if (!blockAdded) + throw new Error('WDB: Block not added.'); + + if (blockAdded.filterUpdated) { + // We remove block, because adding the same block twice, will ignore + // already indexed transactions. This handles the case where single + // transaction has undiscovered outputs. + await this._removeBlock(entry); + + return { + type: scanActions.REPEAT + }; + } + + return { + type: scanActions.NEXT + }; + } + /** * Add a transaction to the database, map addresses * to wallet IDs, potentially store orphans, resolve * orphans, or confirm a transaction. * @param {TX} tx - * @returns {Promise} + * @returns {Promise} */ async addTX(tx) { @@ -2482,7 +2637,7 @@ class WalletDB extends EventEmitter { * @private * @param {TX} tx * @param {BlockMeta} block - * @returns {Promise} + * @returns {Promise} */ async _addTX(tx, block) { @@ -2498,6 +2653,7 @@ class WalletDB extends EventEmitter { wids.size, tx.txid()); let result = false; + let filterUpdated = false; // Insert the transaction // into every matching wallet. @@ -2506,18 +2662,27 @@ class WalletDB extends EventEmitter { assert(wallet); - if (await wallet.add(tx, block)) { + const wadded = await wallet.add(tx, block); + + if (wadded) { + result = true; + + if (wadded.derived.length > 0) + filterUpdated = true; + this.logger.info( 'Added transaction to wallet in WalletDB: %s (%d).', wallet.id, wid); - result = true; } } if (!result) return null; - return wids; + return { + wids, + filterUpdated + }; } /** diff --git a/test/auction-rpc-test.js b/test/auction-rpc-test.js index 9a8092987..d55742655 100644 --- a/test/auction-rpc-test.js +++ b/test/auction-rpc-test.js @@ -23,6 +23,9 @@ class TestUtil { bip37: true, wallet: true }); + + this.nodeCtx.init(); + this.network = this.nodeCtx.network; this.txs = {}; this.blocks = {}; diff --git a/test/node-http-test.js b/test/node-http-test.js index 0bf8c16b5..1fbcca6e8 100644 --- a/test/node-http-test.js +++ b/test/node-http-test.js @@ -28,9 +28,9 @@ describe('Node HTTP', function() { beforeEach(async () => { nodeCtx = new NodeContext(); - nclient = nodeCtx.nclient; await nodeCtx.open(); + nclient = nodeCtx.nclient; }); afterEach(async () => { @@ -74,9 +74,8 @@ describe('Node HTTP', function() { beforeEach(async () => { nodeCtx = new NodeContext(); - nclient = nodeCtx.nclient; - await nodeCtx.open(); + nclient = nodeCtx.nclient; }); afterEach(async () => { @@ -261,9 +260,8 @@ describe('Node HTTP', function() { network: 'regtest' }); - const interval = nodeCtx.network.names.treeInterval; - await nodeCtx.open(); + const interval = nodeCtx.network.names.treeInterval; const nclient = nodeCtx.nclient; const node = nodeCtx.node; @@ -315,9 +313,9 @@ describe('Node HTTP', function() { network: 'regtest' }); + await nodeCtx.open(); const {network, nclient} = nodeCtx; - await nodeCtx.open(); const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -337,9 +335,9 @@ describe('Node HTTP', function() { network: 'regtest', listen: true }); - const {network, nclient} = nodeCtx; await nodeCtx.open(); + const {network, nclient} = nodeCtx; const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -359,9 +357,9 @@ describe('Node HTTP', function() { network: 'main' }); + await nodeCtx.open(); const {network, nclient} = nodeCtx; - await nodeCtx.open(); const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -382,9 +380,9 @@ describe('Node HTTP', function() { listen: true }); + await nodeCtx.open(); const {network, nclient} = nodeCtx; - await nodeCtx.open(); const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -412,9 +410,9 @@ describe('Node HTTP', function() { publicBrontidePort }); + await nodeCtx.open(); const {network, nclient} = nodeCtx; - await nodeCtx.open(); const {pool} = await nclient.getInfo(); assert.strictEqual(pool.host, '0.0.0.0'); @@ -443,6 +441,8 @@ describe('Node HTTP', function() { rejectAbsurdFees: false }); + nodeCtx.init(); + const {network, nclient} = nodeCtx; const {treeInterval} = network.names; diff --git a/test/node-rescan-test.js b/test/node-rescan-test.js index c651a3e57..40cbc5069 100644 --- a/test/node-rescan-test.js +++ b/test/node-rescan-test.js @@ -66,11 +66,10 @@ describe('Node Rescan Interactive API', function() { before(async () => { nodeCtx = new NodeContext(); - const {network} = nodeCtx; - - funderWallet = new MemWallet({ network }); await nodeCtx.open(); + const {network} = nodeCtx; + funderWallet = new MemWallet({ network }); nodeCtx.on('connect', (entry, block) => { funderWallet.addBlock(entry, block.txs); @@ -400,20 +399,57 @@ describe('Node Rescan Interactive API', function() { node.scanInteractive(startHeight, null, getIter(counter2)) ]); - assert.strictEqual(counter1.count, 10); - assert.strictEqual(counter2.count, 10); + assert.strictEqual(counter1.count, RESCAN_DEPTH); + assert.strictEqual(counter2.count, RESCAN_DEPTH); - // Chain gets locked per block, so we should see alternating events. + // Chain gets locked per block by default, so we should see alternating events. // Because they start in parallel, but id1 starts first they will be // getting events in alternating older (first one gets lock, second waits, // second gets lock, first waits, etc.) - for (let i = 0; i < 10; i++) { + for (let i = 0; i < RESCAN_DEPTH; i++) { assert.strictEqual(events[i].id, 1); assert.strictEqual(events[i + 1].id, 2); i++; } }); + it('should rescan in series', async () => { + const {node} = nodeCtx; + const startHeight = nodeCtx.height - RESCAN_DEPTH + 1; + + const events = []; + const getIter = (counterObj) => { + return async (entry, txs) => { + assert.strictEqual(entry.height, startHeight + counterObj.count); + assert.strictEqual(txs.length, 4); + + events.push({ ...counterObj }); + counterObj.count++; + + return { + type: scanActions.NEXT + }; + }; + }; + + const counter1 = { id: 1, count: 0 }; + const counter2 = { id: 2, count: 0 }; + await Promise.all([ + node.scanInteractive(startHeight, null, getIter(counter1), true), + node.scanInteractive(startHeight, null, getIter(counter2), true) + ]); + + assert.strictEqual(counter1.count, RESCAN_DEPTH); + assert.strictEqual(counter2.count, RESCAN_DEPTH); + + // We lock the whole chain for this test, so we should see events + // from one to other. + for (let i = 0; i < RESCAN_DEPTH; i++) { + assert.strictEqual(events[i].id, 1); + assert.strictEqual(events[i + RESCAN_DEPTH].id, 2); + } + }); + describe('HTTP', function() { let client = null; @@ -457,7 +493,7 @@ describe('Node Rescan Interactive API', function() { filter = test.filter.encode(); await client.rescanInteractive(startHeight, filter); - assert.strictEqual(count, 10); + assert.strictEqual(count, RESCAN_DEPTH); count = 0; if (test.filter) @@ -507,7 +543,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) filter = test.filter.encode(); - await client.rescanInteractive(startHeight, filter); + let err; + try { + await client.rescanInteractive(startHeight, filter); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); @@ -518,7 +561,15 @@ describe('Node Rescan Interactive API', function() { if (test.filter) await client.setFilter(test.filter.encode()); - await client.rescanInteractive(startHeight, null); + err = null; + try { + await client.rescanInteractive(startHeight, null); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); }); @@ -560,7 +611,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) filter = test.filter.encode(); - await client.rescanInteractive(startHeight, filter); + let err; + try { + await client.rescanInteractive(startHeight, filter); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); @@ -570,7 +628,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) await client.setFilter(test.filter.encode()); - await client.rescanInteractive(startHeight); + err = null; + try { + await client.rescanInteractive(startHeight); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); }); @@ -612,7 +677,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) filter = test.filter.encode(); - await client.rescanInteractive(startHeight, filter); + let err; + try { + await client.rescanInteractive(startHeight, filter); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); @@ -622,7 +694,14 @@ describe('Node Rescan Interactive API', function() { if (test.filter) await client.setFilter(test.filter.encode()); - await client.rescanInteractive(startHeight); + err = null; + try { + await client.rescanInteractive(startHeight); + } catch (e) { + err = e; + } + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, 5); assert.strictEqual(aborted, true); }); @@ -669,7 +748,15 @@ describe('Node Rescan Interactive API', function() { if (test.filter) filter = test.filter.encode(); - await client.rescanInteractive(startHeight, filter); + let err; + try { + await client.rescanInteractive(startHeight, filter); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(count, tests.length); assert.strictEqual(aborted, true); }); @@ -712,17 +799,33 @@ describe('Node Rescan Interactive API', function() { aborted = true; }); - await client.rescanInteractive(startHeight, filter.encode()); + let err; + try { + await client.rescanInteractive(startHeight, filter.encode()); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(aborted, true); // Now try using client.filter + err = null; aborted = false; filter = BloomFilter.fromRate(10000, 0.001); testTXs = allTXs[startHeight].slice(); expected = 0; await client.setFilter(filter.encode()); - await client.rescanInteractive(startHeight); + try { + await client.rescanInteractive(startHeight); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'scan request aborted.'); assert.strictEqual(aborted, true); }); @@ -758,20 +861,63 @@ describe('Node Rescan Interactive API', function() { client2.rescanInteractive(startHeight) ]); - assert.strictEqual(counter1.count, 10); - assert.strictEqual(counter2.count, 10); + assert.strictEqual(counter1.count, RESCAN_DEPTH); + assert.strictEqual(counter2.count, RESCAN_DEPTH); // Chain gets locked per block, so we should see alternating events. // Because they start in parallel, but id1 starts first they will be // getting events in alternating older (first one gets lock, second waits, // second gets lock, first waits, etc.) - for (let i = 0; i < 10; i++) { + for (let i = 0; i < RESCAN_DEPTH; i++) { assert.strictEqual(events[i].id, 1); assert.strictEqual(events[i + 1].id, 2); i++; } }); + it('should rescan in series', async () => { + const client2 = nodeCtx.nodeClient(); + await client2.open(); + + const startHeight = nodeCtx.height - RESCAN_DEPTH + 1; + const events = []; + const counter1 = { id: 1, count: 0 }; + const counter2 = { id: 2, count: 0 }; + + const getIter = (counterObj) => { + return async (rawEntry, rawTXs) => { + const [entry, txs] = parseBlock(rawEntry, rawTXs); + assert.strictEqual(entry.height, startHeight + counterObj.count); + assert.strictEqual(txs.length, 4); + + events.push({ ...counterObj }); + counterObj.count++; + + return { + type: scanActions.NEXT + }; + }; + }; + + client.hook('block rescan interactive', getIter(counter1)); + client2.hook('block rescan interactive', getIter(counter2)); + + await Promise.all([ + client.rescanInteractive(startHeight, null, true), + client2.rescanInteractive(startHeight, null, true) + ]); + + assert.strictEqual(counter1.count, RESCAN_DEPTH); + assert.strictEqual(counter2.count, RESCAN_DEPTH); + + // We lock the whole chain for this test, so we should see events + // from one to other. + for (let i = 0; i < RESCAN_DEPTH; i++) { + assert.strictEqual(events[i].id, 1); + assert.strictEqual(events[i + RESCAN_DEPTH].id, 2); + } + }); + // Make sure the client closing does not cause the chain locker to get // indefinitely locked. (https://github.com/bcoin-org/bsock/pull/11) it('should stop rescan when client closes', async () => { diff --git a/test/node-rpc-test.js b/test/node-rpc-test.js index 0b1428010..9701d4398 100644 --- a/test/node-rpc-test.js +++ b/test/node-rpc-test.js @@ -37,6 +37,7 @@ describe('RPC', function() { describe('getblockchaininfo', function() { const nodeCtx = new NodeContext(nodeOptions); + nodeCtx.init(); const nclient = nodeCtx.nclient; before(async () => { @@ -58,6 +59,7 @@ describe('RPC', function() { describe('getrawmempool', function() { const nodeCtx = new NodeContext(nodeOptions); + nodeCtx.init(); const nclient = nodeCtx.nclient; before(async () => { @@ -83,10 +85,9 @@ describe('RPC', function() { name: 'node-rpc-test' }); + await nodeCtx.open(); nclient = nodeCtx.nclient; node = nodeCtx.node; - - await nodeCtx.open(); }); after(async () => { @@ -221,6 +222,7 @@ describe('RPC', function() { ...nodeOptions, spv: true }); + await nodeCtx.open(); await assert.rejects(async () => { @@ -265,7 +267,6 @@ describe('RPC', function() { // default - prune: false nodeCtx = new NodeContext(nodeOptions); await nodeCtx.open(); - const {miner, nclient} = nodeCtx; const addr = 'rs1q4rvs9pp9496qawp2zyqpz3s90fjfk362q92vq8'; @@ -320,6 +321,7 @@ describe('RPC', function() { describe('mining', function() { const nodeCtx = new NodeContext(nodeOptions); + nodeCtx.init(); const { miner, chain, @@ -505,6 +507,7 @@ describe('RPC', function() { ...nodeOptions, indexTX: true }); + nodeCtx.init(); const { miner, @@ -588,6 +591,7 @@ describe('RPC', function() { describe('networking', function() { const nodeCtx = new NodeContext({ ...nodeOptions, bip37: true }); + nodeCtx.init(); const nclient = nodeCtx.nclient; before(async () => { @@ -607,6 +611,7 @@ describe('RPC', function() { describe('DNS Utility', function() { const nodeCtx = new NodeContext(nodeOptions); + nodeCtx.init(); const nclient = nodeCtx.nclient; before(async () => { @@ -762,6 +767,8 @@ describe('RPC', function() { wallet: true }); + nodeCtx.init(); + const { node, nclient, diff --git a/test/node-spv-sync-test.js b/test/node-spv-sync-test.js index 62134b2ec..7d411cbbc 100644 --- a/test/node-spv-sync-test.js +++ b/test/node-spv-sync-test.js @@ -158,7 +158,7 @@ describe('SPV Node Sync', function() { }); it('should send a tx from chain 1 to SPV node', async () => { - const balanceEvent = forEvent(spvwallet, 'balance'); + const balanceEvent = forEvent(spvwallet, 'balance', 1, 9000); await wallet.send({ outputs: [{ value: 1012345678, diff --git a/test/util/balance.js b/test/util/balance.js new file mode 100644 index 000000000..d53b612bf --- /dev/null +++ b/test/util/balance.js @@ -0,0 +1,144 @@ +'use strict'; + +const assert = require('bsert'); +const Wallet = require('../../lib/wallet/wallet'); +const WalletClient = require('../../lib/client/wallet'); + +/** + * @property {Number} tx + * @property {Number} coin + * @property {Number} confirmed + * @property {Number} unconfirmed + * @property {Number} ulocked - unconfirmed locked + * @property {Number} clocked - confirmed locked + */ + +class Balance { + constructor(options) { + options = options || {}; + + this.tx = options.tx || 0; + this.coin = options.coin || 0; + this.confirmed = options.confirmed || 0; + this.unconfirmed = options.unconfirmed || 0; + this.ulocked = options.ulocked || 0; + this.clocked = options.clocked || 0; + } + + clone() { + return new Balance(this); + } + + cloneWithDelta(obj) { + return this.clone().apply(obj); + } + + fromBalance(obj) { + this.tx = obj.tx; + this.coin = obj.coin; + this.confirmed = obj.confirmed; + this.unconfirmed = obj.unconfirmed; + this.ulocked = obj.lockedUnconfirmed; + this.clocked = obj.lockedConfirmed; + + return this; + } + + apply(balance) { + this.tx += balance.tx || 0; + this.coin += balance.coin || 0; + this.confirmed += balance.confirmed || 0; + this.unconfirmed += balance.unconfirmed || 0; + this.ulocked += balance.ulocked || 0; + this.clocked += balance.clocked || 0; + + return this; + } + + diff(balance) { + return new Balance({ + tx: this.tx - balance.tx, + coin: this.coin - balance.coin, + confirmed: this.confirmed - balance.confirmed, + unconfirmed: this.unconfirmed - balance.unconfirmed, + ulocked: this.ulocked - balance.ulocked, + clocked: this.clocked - balance.clocked + }); + } + + static fromBalance(wbalance) { + return new this().fromBalance(wbalance); + } +} + +/** + * @param {Wallet} wallet + * @param {String} accountName + * @returns {Promise} + */ + +async function getWalletBalance(wallet, accountName) { + assert(wallet instanceof Wallet); + const balance = await wallet.getBalance(accountName); + return Balance.fromBalance(balance.getJSON(true)); +} + +/** + * @param {WalletClient} wclient + * @param {String} id + * @param {String} accountName + * @returns {Promise} + */ + +async function getWClientBalance(wclient, id, accountName) { + assert(wclient instanceof WalletClient); + const balance = await wclient.getBalance(id, accountName); + return Balance.fromBalance(balance); +} + +/** + * @param {WalletClient.Wallet} balance + * @param {String} accountName + * @returns {Promise} + */ + +async function getWClientWalletBalance(wallet, accountName) { + assert(wallet instanceof WalletClient.Wallet); + const balance = await wallet.getBalance(accountName); + return Balance.fromBalance(balance); +} + +async function getBalance(wallet, accountName) { + if (wallet instanceof WalletClient.Wallet) + return getWClientWalletBalance(wallet, accountName); + + return getWalletBalance(wallet, accountName); +} + +/** + * @param {Wallet} wallet + * @param {String} accountName + * @param {Balance} expectedBalance + * @param {String} message + * @returns {Promise} + */ + +async function assertBalanceEquals(wallet, accountName, expectedBalance, message) { + const balance = await getBalance(wallet, accountName); + assert.deepStrictEqual(balance, expectedBalance, message); +} + +async function assertWClientBalanceEquals(wclient, id, accountName, expectedBalance, message) { + const balance = await getWClientBalance(wclient, id, accountName); + assert.deepStrictEqual(balance, expectedBalance, message); +} + +exports.Balance = Balance; + +exports.getBalance = getBalance; +exports.getWalletBalance = getWalletBalance; +exports.getWClientBalance = getWClientBalance; +exports.getWClientWalletBalance = getWClientWalletBalance; + +exports.assertBalanceEquals = assertBalanceEquals; +exports.assertWClientBalanceEquals = assertWClientBalanceEquals; diff --git a/test/util/common.js b/test/util/common.js index c2a6e3f06..57b738558 100644 --- a/test/util/common.js +++ b/test/util/common.js @@ -103,13 +103,14 @@ common.rimraf = async function(p) { return await fs.rimraf(p); }; -common.forValue = async function forValue(obj, key, val, timeout = 5000) { +common.forValue = async function forValue(obj, key, val, timeout = 2000) { assert(typeof obj === 'object'); assert(typeof key === 'string'); const ms = 10; let interval = null; let count = 0; + const stack = getStack(); return new Promise((resolve, reject) => { interval = setInterval(() => { @@ -118,14 +119,16 @@ common.forValue = async function forValue(obj, key, val, timeout = 5000) { resolve(); } else if (count * ms >= timeout) { clearInterval(interval); - reject(new Error('Timeout waiting for value.')); + const error = new Error('Timeout waiting for value.'); + error.stack = error.stack + '\n' + stack; + reject(error); } count += 1; }, ms); }); }; -common.forEvent = async function forEvent(obj, name, count = 1, timeout = 5000) { +common.forEvent = async function forEvent(obj, name, count = 1, timeout = 2000) { assert(typeof obj === 'object'); assert(typeof name === 'string'); assert(typeof count === 'number'); @@ -134,6 +137,8 @@ common.forEvent = async function forEvent(obj, name, count = 1, timeout = 5000) let countdown = count; const events = []; + const stack = getStack(); + return new Promise((resolve, reject) => { let timeoutHandler, listener; @@ -159,9 +164,11 @@ common.forEvent = async function forEvent(obj, name, count = 1, timeout = 5000) timeoutHandler = setTimeout(() => { cleanup(); const msg = `Timeout waiting for event ${name} ` - + `(received ${count - countdown}/${count})`; + + `(received ${count - countdown}/${count})\n${stack}`; - reject(new Error(msg)); + const error = new Error(msg); + error.stack = error.stack + '\n' + stack; + reject(error); return; }, timeout); @@ -169,12 +176,14 @@ common.forEvent = async function forEvent(obj, name, count = 1, timeout = 5000) }); }; -common.forEventCondition = async function forEventCondition(obj, name, fn, timeout = 5000) { +common.forEventCondition = async function forEventCondition(obj, name, fn, timeout = 2000) { assert(typeof obj === 'object'); assert(typeof name === 'string'); assert(typeof fn === 'function'); assert(typeof timeout === 'number'); + const stack = getStack(); + return new Promise((resolve, reject) => { let timeoutHandler, listener; @@ -190,6 +199,7 @@ common.forEventCondition = async function forEventCondition(obj, name, fn, timeo res = await fn(...args); } catch (e) { cleanup(); + e.stack = e.stack + '\n' + stack; reject(e); return; } @@ -203,7 +213,9 @@ common.forEventCondition = async function forEventCondition(obj, name, fn, timeo timeoutHandler = setTimeout(() => { cleanup(); const msg = `Timeout waiting for event ${name} with condition`; - reject(new Error(msg)); + const error = new Error(msg); + error.stack = error.stack + '\n' + stack; + reject(error); return; }, timeout); @@ -357,3 +369,7 @@ class TXContext { return [tx, view]; } } + +function getStack() { + return new Error().stack.split('\n').slice(2).join('\n'); +} diff --git a/test/util/node-context.js b/test/util/node-context.js index c0a49db1c..aa2efd3eb 100644 --- a/test/util/node-context.js +++ b/test/util/node-context.js @@ -3,10 +3,11 @@ const assert = require('bsert'); const common = require('./common'); const fs = require('bfile'); +const Network = require('../../lib/protocol/network'); const SPVNode = require('../../lib/node/spvnode'); const FullNode = require('../../lib/node/fullnode'); +const WalletNode = require('../../lib/wallet/node'); const plugin = require('../../lib/wallet/plugin'); -const Network = require('../../lib/protocol/network'); const {NodeClient, WalletClient} = require('../../lib/client'); const Logger = require('blgr'); @@ -23,7 +24,7 @@ class NodeContext { constructor(options = {}) { this.name = 'node-test'; this.options = {}; - this.node = null; + this.prefix = null; this.opened = false; this.logger = new Logger({ console: true, @@ -31,13 +32,15 @@ class NodeContext { level: 'none' }); + this.initted = false; + this.node = null; + this.walletNode = null; this.nclient = null; this.wclient = null; this.clients = []; this.fromOptions(options); - this.init(); } fromOptions(options) { @@ -54,6 +57,11 @@ class NodeContext { walletHttpPort: null }; + if (options.name != null) { + assert(typeof options.name === 'string'); + this.name = options.name; + } + if (options.network != null) fnodeOptions.network = Network.get(options.network).type; @@ -66,17 +74,23 @@ class NodeContext { } if (options.prefix != null) { - fnodeOptions.prefix = this.prefix; + fnodeOptions.prefix = options.prefix; fnodeOptions.memory = false; + this.prefix = fnodeOptions.prefix; } if (options.memory != null) { - assert(!fnodeOptions.prefix, 'Can not set prefix with memory.'); + assert(typeof options.memory === 'boolean'); + assert(!(options.memory && options.prefix), + 'Can not set prefix with memory.'); + fnodeOptions.memory = options.memory; } - if (!this.memory && !this.prefix) + if (!fnodeOptions.memory && !fnodeOptions.prefix) { fnodeOptions.prefix = common.testdir(this.name); + this.prefix = fnodeOptions.prefix; + } if (options.wallet != null) fnodeOptions.wallet = options.wallet; @@ -101,23 +115,45 @@ class NodeContext { fnodeOptions.timeout = options.timeout; } + if (options.standalone != null) { + assert(typeof options.standalone === 'boolean'); + fnodeOptions.standalone = options.standalone; + } + this.options = fnodeOptions; } init() { + if (this.initted) + return; + if (this.options.spv) this.node = new SPVNode(this.options); else this.node = new FullNode(this.options); - if (this.options.wallet) + if (this.options.wallet && !this.options.standalone) { this.node.use(plugin); + } else if (this.options.wallet && this.options.standalone) { + this.walletNode = new WalletNode({ + ...this.options, + + nodeHost: '127.0.0.1', + nodePort: this.options.httpPort, + nodeApiKey: this.options.apiKey, + + httpPort: this.options.walletHttpPort, + apiKey: this.options.apiKey + }); + } // Initial wallets. this.nclient = this.nodeClient(); if (this.options.wallet) this.wclient = this.walletClient(); + + this.initted = true; } get network() { @@ -145,6 +181,12 @@ class NodeContext { } get wdb() { + if (!this.options.wallet) + return null; + + if (this.walletNode) + return this.walletNode.wdb; + return this.node.get('walletdb').wdb; } @@ -177,16 +219,26 @@ class NodeContext { */ async open() { + this.init(); + if (this.opened) return; if (this.prefix) await fs.mkdirp(this.prefix); + const open = common.forEvent(this.node, 'open'); await this.node.ensure(); await this.node.open(); await this.node.connect(); this.node.startSync(); + await open; + + if (this.walletNode) { + const walletOpen = common.forEvent(this.walletNode, 'open'); + await this.walletNode.open(); + await walletOpen; + } if (this.wclient) await this.wclient.open(); @@ -200,7 +252,6 @@ class NodeContext { if (!this.opened) return; - const close = common.forEvent(this.node, 'close'); const closeClients = []; for (const client of this.clients) { @@ -209,10 +260,22 @@ class NodeContext { } await Promise.all(closeClients); + + if (this.walletNode) { + const walletClose = common.forEvent(this.walletNode, 'close'); + await this.walletNode.close(); + await walletClose; + } + + const close = common.forEvent(this.node, 'close'); await this.node.close(); await close; + this.node = null; + this.wclient = null; + this.nclient = null; this.opened = false; + this.initted = false; } async destroy() { @@ -224,8 +287,8 @@ class NodeContext { * Helpers */ - enableLogging() { - this.logger.setLevel('debug'); + enableLogging(level = 'debug') { + this.logger.setLevel(level); } disableLogging() { @@ -299,17 +362,20 @@ class NodeContext { * Mine blocks and wait for connect. * @param {Number} count * @param {Address} address + * @param {ChainEntry} [tip=chain.tip] - Tip to mine on * @returns {Promise} - Block hashes */ - async mineBlocks(count, address) { + async mineBlocks(count, address, tip) { assert(this.open); - const blockEvents = common.forEvent(this.node, 'block', count); - const hashes = await this.nodeRPC.generateToAddress([count, address]); - await blockEvents; + if (!tip) + tip = this.chain.tip; - return hashes; + for (let i = 0; i < count; i++) { + const block = await this.miner.mineBlock(tip, address); + tip = await this.chain.add(block); + } } } diff --git a/test/util/nodes-context.js b/test/util/nodes-context.js index 0be0f325c..2989464df 100644 --- a/test/util/nodes-context.js +++ b/test/util/nodes-context.js @@ -21,37 +21,55 @@ class NodesContext { addNode(options = {}) { const index = this.nodeCtxs.length; - let seedPort = this.network.port + index - 1; + let seedPort = getPort(this.network, index - 1); - if (seedPort < this.network.port) - seedPort = this.network.port; + if (options.seedNodeIndex != null) + seedPort = getPort(this.network, options.seedNodeIndex); const port = this.network.port + index; const brontidePort = this.network.brontidePort + index; const httpPort = this.network.rpcPort + index + 100; const walletHttpPort = this.network.walletPort + index + 200; + const nsPort = this.network.nsPort + index; + const rsPort = this.network.rsPort + index + 100; + + const seeds = []; + + if (options.seedNodeIndex != null || index > 0) + seeds.push(`127.0.0.1:${seedPort}`); const nodeCtx = new NodeContext({ + listen: true, + ...options, // override name: `node-${index}`, network: this.network, - listen: true, - publicHost: '127.0.0.1', - publicPort: port, + port: port, brontidePort: brontidePort, + rsPort: rsPort, + nsPort: nsPort, httpPort: httpPort, walletHttpPort: walletHttpPort, - seeds: [ - `127.0.0.1:${seedPort}` - ] + + seeds: seeds }); this.nodeCtxs.push(nodeCtx); + return nodeCtx; } - open() { + /** + * Open all or specific nodes. + * @param {Number} [index=-1] default all + * @returns {Promise} + */ + + open(index = -1) { + if (index !== -1) + return this.context(index).open(); + const jobs = []; for (const nodeCtx of this.nodeCtxs) @@ -60,7 +78,16 @@ class NodesContext { return Promise.all(jobs); } - close() { + /** + * Close all or specific nodes. + * @param {Number} [index=-1] default all + * @returns {Promise} + */ + + close(index = -1) { + if (index !== -1) + return this.context(index).close(); + const jobs = []; for (const nodeCtx of this.nodeCtxs) @@ -69,21 +96,52 @@ class NodesContext { return Promise.all(jobs); } + /** + * Destroy specific or all nodes. Clean up directories on the disk. + * @param {Number} [index=-1] default all + * @returns {Promise} + */ + + destroy(index = -1) { + if (index !== -1) + return this.context(index).destroy(); + + const jobs = []; + + for (const nodeCtx of this.nodeCtxs) + jobs.push(nodeCtx.destroy()); + + return Promise.all(jobs); + } + + /** + * Connect all nodes. + * @returns {Promise} + */ + async connect() { for (const nodeCtx of this.nodeCtxs) { await nodeCtx.node.connect(); - await new Promise(r => setTimeout(r, 1000)); + await nodeCtx.node.startSync(); } } + /** + * Disconnect all nodes. + * @returns {Promise} + */ + async disconnect() { for (let i = this.nodeCtxs.length - 1; i >= 0; i--) { - const node = this.nodeCtxs[i]; + const node = this.nodeCtxs[i].node; await node.disconnect(); - await new Promise(r => setTimeout(r, 1000)); } } + /** + * Start syncing. + */ + startSync() { for (const nodeCtx of this.nodeCtxs) { nodeCtx.chain.synced = true; @@ -92,33 +150,53 @@ class NodesContext { } } + /** + * Stop syncing. + */ + stopSync() { for (const nodeCtx of this.nodeCtxs) nodeCtx.stopSync(); } - async generate(index, blocks) { - const nodeCtx = this.nodeCtxs[index]; - - assert(nodeCtx); - - for (let i = 0; i < blocks; i++) { - const block = await nodeCtx.miner.mineBlock(); - await nodeCtx.chain.add(block); - } + /** + * Mine blocks. + * @param {Number} index + * @param {Number} blocks + * @param {String} address + * @param {ChainEntry} [tip=chain.tip] + * @returns {Promise} + */ + + async generate(index, blocks, address, tip) { + return this.context(index).mineBlocks(blocks, address, tip); } + /** + * Get NodeCtx for the node. + * @param {Number} index + * @returns {NodeContext} + */ + context(index) { - const node = this.nodeCtxs[index]; - assert(node); - return node; + const nodeCtx = this.nodeCtxs[index]; + assert(nodeCtx); + return nodeCtx; } + /** + * Get height for the node. + * @param {Number} index + * @returns {Number} + */ + height(index) { - const nodeCtx = this.nodeCtxs[index]; - assert(nodeCtx); - return nodeCtx.height; + return this.context(index).height; } } +function getPort(network, index) { + return Math.max(network.port + index, network.port); +} + module.exports = NodesContext; diff --git a/test/util/wallet.js b/test/util/wallet.js index 812ea34de..255024790 100644 --- a/test/util/wallet.js +++ b/test/util/wallet.js @@ -1,18 +1,20 @@ 'use strict'; +const assert = require('bsert'); const blake2b = require('bcrypto/lib/blake2b'); const random = require('bcrypto/lib/random'); -const Block = require('../../lib/primitives/block'); const ChainEntry = require('../../lib/blockchain/chainentry'); const Input = require('../../lib/primitives/input'); const Outpoint = require('../../lib/primitives/outpoint'); +const {ZERO_HASH} = require('../../lib/protocol/consensus'); const walletUtils = exports; -walletUtils.fakeBlock = (height) => { - const prev = blake2b.digest(fromU32((height - 1) >>> 0)); - const hash = blake2b.digest(fromU32(height >>> 0)); - const root = blake2b.digest(fromU32((height | 0x80000000) >>> 0)); +walletUtils.fakeBlock = (height, prevSeed = 0, seed = prevSeed) => { + assert(height >= 0); + const prev = height === 0 ? ZERO_HASH : blake2b.digest(fromU32(((height - 1) ^ prevSeed) >>> 0)); + const hash = blake2b.digest(fromU32((height ^ seed) >>> 0)); + const root = blake2b.digest(fromU32((height | 0x80000000 ^ seed) >>> 0)); return { hash: hash, @@ -36,22 +38,26 @@ walletUtils.dummyInput = () => { return Input.fromOutpoint(new Outpoint(hash, 0)); }; -walletUtils.nextBlock = (wdb) => { - return walletUtils.fakeBlock(wdb.state.height + 1); +walletUtils.nextBlock = (wdb, prevSeed = 0, seed = prevSeed) => { + return walletUtils.fakeBlock(wdb.state.height + 1, prevSeed, seed); }; -walletUtils.curBlock = (wdb) => { - return walletUtils.fakeBlock(wdb.state.height); +walletUtils.curBlock = (wdb, prevSeed = 0, seed = prevSeed) => { + return walletUtils.fakeBlock(wdb.state.height, prevSeed, seed); }; -walletUtils.nextEntry = (wdb) => { - const cur = walletUtils.curEntry(wdb); - const next = new Block(walletUtils.nextBlock(wdb)); - return ChainEntry.fromBlock(next, cur); +walletUtils.fakeEntry = (height, prevSeed = 0, curSeed = prevSeed) => { + const cur = walletUtils.fakeBlock(height, prevSeed, curSeed); + return new ChainEntry(cur);; }; -walletUtils.curEntry = (wdb) => { - return new ChainEntry(walletUtils.curBlock(wdb)); +walletUtils.nextEntry = (wdb, curSeed = 0, nextSeed = curSeed) => { + const next = walletUtils.nextBlock(wdb, curSeed, nextSeed); + return new ChainEntry(next); +}; + +walletUtils.curEntry = (wdb, prevSeed = 0, seed = prevSeed) => { + return walletUtils.fakeEntry(wdb.state.height, seed); }; function fromU32(num) { diff --git a/test/wallet-auction-test.js b/test/wallet-auction-test.js index 267e111dd..ece157689 100644 --- a/test/wallet-auction-test.js +++ b/test/wallet-auction-test.js @@ -131,8 +131,9 @@ describe('Wallet Auction', function() { const openMTX = openTXs[openIndex++]; const tx = openMTX.toTX(); const addResult = await wdb.addTX(tx); - assert.strictEqual(addResult.size, 1); - assert.ok(addResult.has(wallet.wid)); + assert.ok(addResult); + assert.strictEqual(addResult.wids.size, 1); + assert.ok(addResult.wids.has(wallet.wid)); const pending = await wallet.getPending(); assert.strictEqual(pending.length, 1); @@ -305,7 +306,8 @@ describe('Wallet Auction', function() { // double opens are properly removed. await wallet.sign(spendMTX); const added = await wdb.addTX(spendMTX.toTX()); - assert.strictEqual(added.size, 1); + assert.ok(added); + assert.strictEqual(added.wids.size, 1); }); it('should mine enough blocks to expire auction (again)', async () => { diff --git a/test/wallet-balance-test.js b/test/wallet-balance-test.js index 66b5902f5..065f8c99c 100644 --- a/test/wallet-balance-test.js +++ b/test/wallet-balance-test.js @@ -11,6 +11,7 @@ const Output = require('../lib/primitives/output'); const {Resource} = require('../lib/dns/resource'); const {types, grindName} = require('../lib/covenants/rules'); const {forEventCondition} = require('./util/common'); +const {Balance, assertBalanceEquals} = require('./util/balance'); /** * Wallet balance tracking tests. @@ -68,65 +69,9 @@ const DEFAULT_ACCOUNT = 'default'; const ALT_ACCOUNT = 'alt'; // Balances -/** - * @property {Number} tx - * @property {Number} coin - * @property {Number} confirmed - * @property {Number} unconfirmed - * @property {Number} ulocked - unconfirmed locked - * @property {Number} clocked - confirmed locked - */ - -class BalanceObj { - constructor(options) { - options = options || {}; - - this.tx = options.tx || 0; - this.coin = options.coin || 0; - this.confirmed = options.confirmed || 0; - this.unconfirmed = options.unconfirmed || 0; - this.ulocked = options.ulocked || 0; - this.clocked = options.clocked || 0; - } - - clone() { - return new BalanceObj(this); - } - - cloneWithDelta(obj) { - return this.clone().apply(obj); - } - - fromBalance(obj) { - this.tx = obj.tx; - this.coin = obj.coin; - this.confirmed = obj.confirmed; - this.unconfirmed = obj.unconfirmed; - this.ulocked = obj.lockedUnconfirmed; - this.clocked = obj.lockedConfirmed; - - return this; - } - - apply(balance) { - this.tx += balance.tx || 0; - this.coin += balance.coin || 0; - this.confirmed += balance.confirmed || 0; - this.unconfirmed += balance.unconfirmed || 0; - this.ulocked += balance.ulocked || 0; - this.clocked += balance.clocked || 0; - - return this; - } - - static fromBalance(wbalance) { - return new this().fromBalance(wbalance); - } -} - const INIT_BLOCKS = treeInterval; const INIT_FUND = 10e6; -const NULL_BALANCE = new BalanceObj({ +const NULL_BALANCE = new Balance({ tx: 0, coin: 0, unconfirmed: 0, @@ -135,7 +80,7 @@ const NULL_BALANCE = new BalanceObj({ clocked: 0 }); -const INIT_BALANCE = new BalanceObj({ +const INIT_BALANCE = new Balance({ tx: 1, coin: 1, unconfirmed: INIT_FUND, @@ -202,38 +147,22 @@ async function resign(wallet, mtx) { */ /** - * @returns {Promise} + * @returns {Promise} */ -async function getBalanceObj(wallet, accountName) { - const balance = await wallet.getBalance(accountName); - return BalanceObj.fromBalance(balance.getJSON(true)); -} - -async function assertBalance(wallet, accountName, expected, message) { - const balance = await getBalanceObj(wallet, accountName); - assert.deepStrictEqual(balance, expected, message); - - // recalculate balance test - await wallet.recalculateBalances(); - const balance2 = await getBalanceObj(wallet, accountName); - assert.deepStrictEqual(balance2, expected, message); -} - -async function assertRecalcBalance(wallet, accountName, expected, message) { +async function assertRecalcBalanceEquals(wallet, accountName, expected, message) { await wallet.recalculateBalances(); - const balance = await getBalanceObj(wallet, accountName); - assert.deepStrictEqual(balance, expected, message); + assertBalanceEquals(wallet, accountName, expected, message); } /** - * @param {BalanceObj} balance - * @param {BalanceObj} delta - * @returns {BalanceObj} + * @param {Balance} balance + * @param {Balance} delta + * @returns {Balance} */ function applyDelta(balance, delta) { - return balance.clone().apply(delta); + return balance.cloneWithDelta(delta); } describe('Wallet Balance', function() { @@ -364,14 +293,14 @@ describe('Wallet Balance', function() { /** * @typedef {Object} TestBalances - * @property {BalanceObj} TestBalances.initialBalance - * @property {BalanceObj} TestBalances.sentBalance - * @property {BalanceObj} TestBalances.confirmedBalance - * @property {BalanceObj} TestBalances.unconfirmedBalance - * @property {BalanceObj} TestBalances.eraseBalance - * @property {BalanceObj} TestBalances.blockConfirmedBalance - * @property {BalanceObj} TestBalances.blockUnconfirmedBalance - * @property {BalanceObj} [TestBalances.blockFinalConfirmedBalance] + * @property {Balance} TestBalances.initialBalance + * @property {Balance} TestBalances.sentBalance + * @property {Balance} TestBalances.confirmedBalance + * @property {Balance} TestBalances.unconfirmedBalance + * @property {Balance} TestBalances.eraseBalance + * @property {Balance} TestBalances.blockConfirmedBalance + * @property {Balance} TestBalances.blockUnconfirmedBalance + * @property {Balance} [TestBalances.blockFinalConfirmedBalance] */ /** @@ -481,14 +410,14 @@ describe('Wallet Balance', function() { for (const [key, [balanceName, name]] of Object.entries(BALANCE_CHECK_MAP)) { checks[key] = async (wallet) => { - await assertBalance( + await assertBalanceEquals( wallet, DEFAULT_ACCOUNT, defBalances[balanceName], `${name} balance is incorrect in the account ${DEFAULT_ACCOUNT}.` ); - await assertRecalcBalance( + await assertRecalcBalanceEquals( wallet, DEFAULT_ACCOUNT, defBalances[balanceName], @@ -497,14 +426,14 @@ describe('Wallet Balance', function() { ); if (altBalances != null) { - await assertBalance( + await assertBalanceEquals( wallet, ALT_ACCOUNT, altBalances[balanceName], `${name} balance is incorrect in the account ${ALT_ACCOUNT}.` ); - await assertRecalcBalance( + await assertRecalcBalanceEquals( wallet, ALT_ACCOUNT, altBalances[balanceName], @@ -513,14 +442,14 @@ describe('Wallet Balance', function() { ); } - await assertBalance( + await assertBalanceEquals( wallet, -1, walletBalances[balanceName], `${name} balance is incorrect for the wallet.` ); - await assertRecalcBalance( + await assertRecalcBalanceEquals( wallet, -1, walletBalances[balanceName], diff --git a/test/wallet-coinselection-test.js b/test/wallet-coinselection-test.js index f4cf9e940..ab8d13c41 100644 --- a/test/wallet-coinselection-test.js +++ b/test/wallet-coinselection-test.js @@ -7,11 +7,30 @@ const { WalletDB, policy } = require('..'); +const {BlockMeta} = require('../lib/wallet/records'); // Use main instead of regtest because (deprecated) // CoinSelector.MAX_FEE was network agnostic const network = Network.get('main'); +function dummyBlock(tipHeight) { + const height = tipHeight + 1; + const hash = Buffer.alloc(32); + hash.writeUInt16BE(height); + + const prevHash = Buffer.alloc(32); + prevHash.writeUInt16BE(tipHeight); + + const dummyBlock = { + hash, + height, + time: Date.now(), + prevBlock: prevHash + }; + + return dummyBlock; +} + async function fundWallet(wallet, amounts) { assert(Array.isArray(amounts)); @@ -21,15 +40,8 @@ async function fundWallet(wallet, amounts) { mtx.addOutput(addr, amt); } - const height = wallet.wdb.height + 1; - const hash = Buffer.alloc(32); - hash.writeUInt16BE(height); - const dummyBlock = { - hash, - height, - time: Date.now() - }; - await wallet.wdb.addBlock(dummyBlock, [mtx.toTX()]); + const dummy = dummyBlock(wallet.wdb.height); + await wallet.wdb.addBlock(dummy, [mtx.toTX()]); } describe('Wallet Coin Selection', function () { @@ -41,6 +53,10 @@ describe('Wallet Coin Selection', function () { await wdb.open(); wdb.height = network.txStart + 1; wdb.state.height = wdb.height; + + const dummy = dummyBlock(network.txStart + 1); + const record = BlockMeta.fromEntry(dummy); + await wdb.setTip(record); wallet = wdb.primary; }); diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js index 0be8a1e1a..0048126e7 100644 --- a/test/wallet-rescan-test.js +++ b/test/wallet-rescan-test.js @@ -2,8 +2,15 @@ const assert = require('bsert'); const Network = require('../lib/protocol/network'); -const NodeContext = require('./util/node-context'); +const Address = require('../lib/primitives/address'); +const HDPublicKey = require('../lib/hd/public'); +const NodesContext = require('./util/nodes-context'); +const {forEvent, forEventCondition} = require('./util/common'); +const {Balance, getWClientBalance, getBalance} = require('./util/balance'); +// Definitions: +// Gapped txs/addresses - addresses with lookahead + 1 gap when deriving. +// // Setup: // - Standalone Node (no wallet) responsible for progressing network. // - Wallet Node (with wallet) responsible for rescanning. @@ -22,60 +29,865 @@ const NodeContext = require('./util/node-context'); // recovery is impossible. This tests situation where in block // derivation depth is lower than wallet lookahead. -// TODO: Rewrite using util/node from the interactive rescan test. -// TODO: Add the standalone Wallet variation. -// TODO: Add initial rescan test. +const combinations = [ + { SPV: false, STANDALONE: false, name: 'Full/Plugin' }, + { SPV: false, STANDALONE: true, name: 'Full/Standalone' }, + { SPV: true, STANDALONE: false, name: 'SPV/Plugin' } + // Not supported. + // { SPV: true, STANDALONE: true, name: 'SPV/Standalone' } +]; + +const noSPVcombinations = combinations.filter(c => !c.SPV); +const regtest = Network.get('regtest'); + +describe('Wallet rescan/addBlock', function() { + for (const {SPV, STANDALONE, name} of noSPVcombinations) { + describe(`rescan/addBlock gapped addresses (${name} Integration)`, function() { + this.timeout(5000); + const TEST_LOOKAHEAD = 20; + + const MAIN = 0; + const TEST_ADDBLOCK = 1; + const TEST_RESCAN = 2; + + const WALLET_NAME = 'test'; + const ACCOUNT = 'default'; + + const regtest = Network.get('regtest'); + + /** @type {NodesContext} */ + let nodes; + let minerWallet, minerAddress; + let main, addBlock, rescan; + + before(async () => { + // Initial node is the one that progresses the network. + nodes = new NodesContext(regtest, 1); + // MAIN_WALLET = 0 + nodes.init({ + wallet: true, + standalone: true, + memory: true, + noDNS: true + }); + + // Add the testing node. + // TEST_ADDBLOCK = 1 + nodes.addNode({ + spv: SPV, + wallet: true, + memory: true, + standalone: STANDALONE, + noDNS: true + }); + + // Add the rescan test node. + // TEST_RESCAN = 2 + nodes.addNode({ + spv: SPV, + wallet: true, + memory: true, + standalone: STANDALONE, + noDNS: true + }); + + await nodes.open(); + + const mainWClient = nodes.context(MAIN).wclient; + minerWallet = nodes.context(MAIN).wclient.wallet('primary'); + minerAddress = (await minerWallet.createAddress('default')).address; + + const mainWallet = await mainWClient.createWallet(WALLET_NAME, { + lookahead: TEST_LOOKAHEAD + }); + assert(mainWallet); + + const master = await mainWClient.getMaster(WALLET_NAME); + + const addBlockWClient = nodes.context(TEST_ADDBLOCK).wclient; + const addBlockWalletResult = await addBlockWClient.createWallet(WALLET_NAME, { + lookahead: TEST_LOOKAHEAD, + mnemonic: master.mnemonic.phrase + }); + assert(addBlockWalletResult); + + const rescanWClient = nodes.context(TEST_RESCAN).wclient; + const rescanWalletResult = await rescanWClient.createWallet(WALLET_NAME, { + lookahead: TEST_LOOKAHEAD, + mnemonic: master.mnemonic.phrase + }); + assert(rescanWalletResult); + + main = {}; + main.client = mainWClient.wallet(WALLET_NAME); + await main.client.open(); + main.wdb = nodes.context(MAIN).wdb; + + addBlock = {}; + addBlock.client = addBlockWClient.wallet(WALLET_NAME); + await addBlock.client.open(); + addBlock.wdb = nodes.context(TEST_ADDBLOCK).wdb; + + rescan = {}; + rescan.client = rescanWClient.wallet(WALLET_NAME); + await rescan.client.open(); + rescan.wdb = nodes.context(TEST_RESCAN).wdb; + + await nodes.generate(MAIN, 10, minerAddress); + }); + + after(async () => { + await nodes.close(); + await nodes.destroy(); + }); + + // Prepare for the rescan and addBlock tests. + it('should send gapped txs on each block', async () => { + const expectedRescanBalance = await getBalance(main.client, ACCOUNT); + const height = nodes.height(MAIN); + const blocks = 5; + + // 1 address per block, all of them gapped. + // Start after first gap, make sure rescan has no clue. + const all = await generateGappedAddresses(main.client, blocks + 1, regtest); + await deriveAddresses(main.client, all[all.length - 1].depth); + const addresses = all.slice(1); + // give addBlock first address. + await deriveAddresses(addBlock.client, addresses[0].depth - TEST_LOOKAHEAD); + + const condFn = entry => entry.height === blocks + height; + const mainWalletBlocks = forEventCondition(main.wdb, 'block connect', condFn); + const addBlockWalletBlocks = forEventCondition(addBlock.wdb, 'block connect', condFn); + const rescanWalletBlocks = forEventCondition(rescan.wdb, 'block connect', condFn); + + for (let i = 0; i < blocks; i++) { + await minerWallet.send({ + outputs: [{ + address: addresses[i].address.toString(regtest), + value: 1e6 + }] + }); + + await nodes.generate(MAIN, 1, minerAddress); + } + + await Promise.all([ + mainWalletBlocks, + addBlockWalletBlocks, + rescanWalletBlocks + ]); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedRescanBalance); + // before the rescan test. + await deriveAddresses(rescan.client, addresses[0].depth - TEST_LOOKAHEAD); + }); + + it('should receive gapped txs on each block (addBlock)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const addBlockBalance = await getBalance(addBlock.client, ACCOUNT); + assert.deepStrictEqual(addBlockBalance, expectedBalance); + + const mainInfo = await main.client.getAccount(ACCOUNT); + const addBlockInfo = await addBlock.client.getAccount(ACCOUNT); + assert.deepStrictEqual(addBlockInfo, mainInfo); + }); + + it('should receive gapped txs on each block (rescan)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const expectedInfo = await main.client.getAccount(ACCOUNT); + + // give rescan first address. + await rescan.wdb.rescan(0); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedBalance); + + const rescanInfo = await rescan.client.getAccount(ACCOUNT); + assert.deepStrictEqual(rescanInfo, expectedInfo); + }); + + it('should send gapped txs in the same block', async () => { + const expectedRescanBalance = await getBalance(rescan.client, ACCOUNT); + const txCount = 5; + + const all = await generateGappedAddresses(main.client, txCount + 1, regtest); + await deriveAddresses(main.client, all[all.length - 1].depth); + const addresses = all.slice(1); + + // give addBlock first address. + await deriveAddresses(addBlock.client, addresses[0].depth - TEST_LOOKAHEAD); + + const mainWalletBlocks = forEvent(main.wdb, 'block connect'); + const addBlockWalletBlocks = forEvent(addBlock.wdb, 'block connect'); + const rescanWalletBlocks = forEvent(rescan.wdb, 'block connect'); + + for (const {address} of addresses) { + await minerWallet.send({ + outputs: [{ + address: address.toString(regtest), + value: 1e6 + }] + }); + } + + await nodes.generate(MAIN, 1, minerAddress); + + await Promise.all([ + mainWalletBlocks, + addBlockWalletBlocks, + rescanWalletBlocks + ]); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedRescanBalance); + + await deriveAddresses(rescan.client, addresses[0].depth - TEST_LOOKAHEAD); + }); -describe('Wallet rescan', function() { - const network = Network.get('regtest'); + it.skip('should receive gapped txs in the same block (addBlock)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const addBlockBalance = await getBalance(addBlock.client, ACCOUNT); + assert.deepStrictEqual(addBlockBalance, expectedBalance); - describe('Deadlock', function() { - const nodeCtx = new NodeContext({ - memory: true, - network: 'regtest', - wallet: true + const mainInfo = await main.client.getAccount(ACCOUNT); + const addBlockInfo = await addBlock.client.getAccount(ACCOUNT); + assert.deepStrictEqual(addBlockInfo, mainInfo); }); - let address, node, wdb; + it('should receive gapped txs in the same block (rescan)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const expectedInfo = await main.client.getAccount(ACCOUNT); + + await rescan.wdb.rescan(0); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedBalance); + + const rescanInfo = await rescan.client.getAccount(ACCOUNT); + assert.deepStrictEqual(rescanInfo, expectedInfo); + }); + + it('should send gapped outputs in the same tx', async () => { + const expectedRescanBalance = await getBalance(rescan.client, ACCOUNT); + const outCount = 5; + + const all = await generateGappedAddresses(main.client, outCount + 1, regtest); + await deriveAddresses(main.client, all[all.length - 1].depth); + const addresses = all.slice(1); + + // give addBlock first address. + await deriveAddresses(addBlock.client, addresses[0].depth - TEST_LOOKAHEAD); + + const mainWalletBlocks = forEvent(main.wdb, 'block connect'); + const addBlockWalletBlocks = forEvent(addBlock.wdb, 'block connect'); + const rescanWalletBlocks = forEvent(rescan.wdb, 'block connect'); + + const outputs = addresses.map(({address}) => ({ + address: address.toString(regtest), + value: 1e6 + })); + + await minerWallet.send({outputs}); + await nodes.generate(MAIN, 1, minerAddress); + + await Promise.all([ + mainWalletBlocks, + addBlockWalletBlocks, + rescanWalletBlocks + ]); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedRescanBalance); + + await deriveAddresses(rescan.client, addresses[0].depth - TEST_LOOKAHEAD); + }); + + it.skip('should receive gapped outputs in the same tx (addBlock)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const addBlockBalance = await getBalance(addBlock.client, ACCOUNT); + assert.deepStrictEqual(addBlockBalance, expectedBalance); + + const mainInfo = await main.client.getAccount(ACCOUNT); + const addBlockInfo = await addBlock.client.getAccount(ACCOUNT); + assert.deepStrictEqual(addBlockInfo, mainInfo); + }); + + it('should receive gapped outputs in the same tx (rescan)', async () => { + const expectedBalance = await getBalance(main.client, ACCOUNT); + const expectedInfo = await main.client.getAccount(ACCOUNT); + + await rescan.wdb.rescan(0); + + const rescanBalance = await getBalance(rescan.client, ACCOUNT); + assert.deepStrictEqual(rescanBalance, expectedBalance); + + const rescanInfo = await rescan.client.getAccount(ACCOUNT); + assert.deepStrictEqual(rescanInfo, expectedInfo); + }); + }); + } + + for (const {SPV, STANDALONE, name} of combinations) { + describe(`Initial sync/rescan (${name} Integration)`, function() { + // Test wallet plugin/standalone is disabled and re-enabled after some time: + // 1. Normal received blocks. + // 2. Reorged after wallet was closed. + // NOTE: Node is not closed, only wallet. + + const MINER = 0; + const WALLET = 1; + const WALLET_NO_WALLET = 2; + + /** @type {NodesContext} */ + let nodes; + let wnodeCtx, noWnodeCtx; + let minerWallet, minerAddress; + let testWallet, testAddress; before(async () => { + nodes = new NodesContext(regtest, 1); + + // MINER = 0 + nodes.init({ + wallet: true, + noDNS: true, + bip37: true + }); + + // WALLET = 1 + wnodeCtx = nodes.addNode({ + noDNS: true, + wallet: true, + + standalone: STANDALONE, + spv: SPV, + + // We need to store on disk in order to test + // recovery on restart + memory: false + }); + + // WALLET_NO_WALLET = 2 + // Wallet node that uses same chain above one + // just does not start wallet. + noWnodeCtx = nodes.addNode({ + noDNS: true, + wallet: false, + prefix: wnodeCtx.prefix, + memory: false, + spv: SPV + }); + + // only open two at a time. + await nodes.open(MINER); + await nodes.open(WALLET); + + minerWallet = nodes.context(MINER).wclient.wallet('primary'); + minerAddress = (await minerWallet.createAddress('default')).address; + + testWallet = wnodeCtx.wclient.wallet('primary'); + testAddress = (await testWallet.createAddress('default')).address; + + await nodes.close(WALLET); + }); + + after(async () => { + await nodes.close(); + await nodes.destroy(); + }); + + afterEach(async () => { + await nodes.close(WALLET); + await nodes.close(WALLET_NO_WALLET); + }); + + it('should fund and spend to wallet', async () => { + await wnodeCtx.open(); + + const txEvent = forEvent(wnodeCtx.wdb, 'tx'); + + // fund wallet. + await nodes.generate(MINER, 9, minerAddress); + + // Send TX to the test wallet. + await minerWallet.send({ + outputs: [{ + address: testAddress, + value: 1e6 + }] + }); + + await nodes.generate(MINER, 1, minerAddress); + await txEvent; + + const balance = await getWClientBalance(wnodeCtx.wclient, 'primary', 'default'); + assert.deepStrictEqual(balance, new Balance({ + coin: 1, + tx: 1, + confirmed: 1e6, + unconfirmed: 1e6 + })); + }); + + it('should rescan/resync after wallet was off', async () => { + // replace wallet node with new one w/o wallet. + await noWnodeCtx.open(); + + await nodes.generate(MINER, 10, minerAddress); + + // Mine in the last block that we will be reorging. + await minerWallet.send({ + outputs: [{ + address: testAddress, + value: 2e6 + }] + }); + + const waitHeight = nodes.height(MINER) + 1; + const nodeSync = forEventCondition(noWnodeCtx.node, 'connect', (entry) => { + return entry.height === waitHeight; + }); + + await nodes.generate(MINER, 1, minerAddress); + await nodeSync; + + // Disable wallet + await noWnodeCtx.close(); + + wnodeCtx.init(); + + const eventsToWait = []; + // For spv we don't wait for sync done, as it will do the full rescan + // and reset the SPVNode as well. It does not depend on the accumulated + // blocks. + if (SPV) { + // This will happen right away, as scan will just call reset + eventsToWait.push(forEvent(wnodeCtx.wdb, 'sync done')); + // This is what matters for the rescan. + eventsToWait.push(forEventCondition(wnodeCtx.wdb, 'block connect', (entry) => { + return entry.height === nodes.height(MINER); + })); + // Make sure node gets resets. + eventsToWait.push(forEvent(wnodeCtx.node, 'reset')); + } else { + eventsToWait.push(forEvent(wnodeCtx.wdb, 'sync done')); + } + + await wnodeCtx.open(); + await Promise.all(eventsToWait); + assert.strictEqual(wnodeCtx.wdb.height, nodes.height(MINER)); + + const balance = await getWClientBalance(wnodeCtx.wclient, 'primary', 'default'); + assert.deepStrictEqual(balance, new Balance({ + coin: 2, + tx: 2, + confirmed: 1e6 + 2e6, + unconfirmed: 1e6 + 2e6 + })); + + await wnodeCtx.close(); + }); + + it('should rescan/resync after wallet was off and node reorged', async () => { + const minerCtx = nodes.context(MINER); + + await noWnodeCtx.open(); + + // Reorg the network + const tip = minerCtx.chain.tip; + const block = await minerCtx.chain.getBlock(tip.hash); + + // Last block contained our tx from previous test. (integration) + assert.strictEqual(block.txs.length, 2); + + const reorgEvent = forEvent(minerCtx.node, 'reorganize'); + const forkTip = await minerCtx.chain.getPrevious(tip); + + // REORG + await nodes.generate(MINER, 2, minerAddress, forkTip); + // Reset mempool/Get rid of tx after reorg. + await nodes.context(MINER).mempool.reset(); + await nodes.generate(MINER, 2, minerAddress); + await reorgEvent; + + // Send another tx, with different output. + await minerWallet.send({ + outputs: [{ + address: testAddress, + value: 3e6 + }] + }); + + const waitHeight = nodes.height(MINER) + 1; + const nodeSync = forEventCondition(noWnodeCtx.node, 'connect', (entry) => { + return entry.height === waitHeight; + }); + + await nodes.generate(MINER, 1, minerAddress); + await nodeSync; + + await noWnodeCtx.close(); + + wnodeCtx.init(); + + // initial sync + const eventsToWait = []; + + if (SPV) { + // This will happen right away, as scan will just call reset + eventsToWait.push(forEvent(wnodeCtx.wdb, 'sync done')); + + // This is what matters for the rescan. + eventsToWait.push(forEventCondition(wnodeCtx.wdb, 'block connect', (entry) => { + return entry.height === nodes.height(MINER); + })); + + // Make sure node gets resets. + eventsToWait.push(forEvent(wnodeCtx.node, 'reset')); + eventsToWait.push(forEvent(wnodeCtx.wdb, 'unconfirmed')); + } else { + eventsToWait.push(forEvent(wnodeCtx.wdb, 'sync done')); + eventsToWait.push(forEvent(wnodeCtx.wdb, 'unconfirmed')); + } + await wnodeCtx.open(); + await Promise.all(eventsToWait); + + assert.strictEqual(wnodeCtx.height, nodes.height(MINER)); + assert.strictEqual(wnodeCtx.wdb.state.height, wnodeCtx.height); + + const balance = await getWClientBalance(wnodeCtx.wclient, 'primary', 'default'); + + // previous transaction should get unconfirmed. + assert.deepStrictEqual(balance, new Balance({ + coin: 3, + tx: 3, + confirmed: 1e6 + 3e6, + unconfirmed: 1e6 + 2e6 + 3e6 + })); + + await wnodeCtx.close(); + }); + + it('should rescan/resync after wallet was off and received gapped txs in the same block', async () => { + if (SPV) + this.skip(); + + const txCount = 5; + await wnodeCtx.open(); + const startingBalance = await getBalance(testWallet, 'default'); + const all = await generateGappedAddresses(testWallet, txCount, regtest); + await wnodeCtx.close(); + + await noWnodeCtx.open(); + + for (const {address} of all) { + await minerWallet.send({ + outputs: [{ + address: address.toString(regtest), + value: 1e6 + }] + }); + } + + const waitHeight = nodes.height(MINER) + 1; + const nodeSync = forEventCondition(noWnodeCtx.node, 'connect', (entry) => { + return entry.height === waitHeight; + }); + + await nodes.generate(MINER, 1, minerAddress); + + await nodeSync; + await noWnodeCtx.close(); + + wnodeCtx.init(); + + const syncDone = forEvent(wnodeCtx.wdb, 'sync done'); + await wnodeCtx.open(); + await syncDone; + assert.strictEqual(wnodeCtx.wdb.height, nodes.height(MINER)); + + const balance = await getBalance(testWallet, 'default'); + const diff = balance.diff(startingBalance); + assert.deepStrictEqual(diff, new Balance({ + tx: txCount, + coin: txCount, + confirmed: 1e6 * txCount, + unconfirmed: 1e6 * txCount + })); + + await wnodeCtx.close(); + }); + + it('should rescan/resync after wallet was off and received gapped coins in the same tx', async () => { + if (SPV) + this.skip(); + + const outCount = 5; + await wnodeCtx.open(); + const startingBalance = await getBalance(testWallet, 'default'); + const all = await generateGappedAddresses(testWallet, outCount, regtest); + await wnodeCtx.close(); + + await noWnodeCtx.open(); + + const outputs = all.map(({address}) => ({ + address: address.toString(regtest), + value: 1e6 + })); + + await minerWallet.send({outputs}); + + const waitHeight = nodes.height(MINER) + 1; + const nodeSync = forEventCondition(noWnodeCtx.node, 'connect', (entry) => { + return entry.height === waitHeight; + }); + + await nodes.generate(MINER, 1, minerAddress); + + await nodeSync; + await noWnodeCtx.close(); + + wnodeCtx.init(); + + const syncDone = forEvent(wnodeCtx.wdb, 'sync done'); + await wnodeCtx.open(); + await syncDone; + assert.strictEqual(wnodeCtx.wdb.height, nodes.height(MINER)); + + const balance = await getBalance(testWallet, 'default'); + const diff = balance.diff(startingBalance); + assert.deepStrictEqual(diff, new Balance({ + tx: 1, + coin: outCount, + confirmed: 1e6 * outCount, + unconfirmed: 1e6 * outCount + })); + + await wnodeCtx.close(); + }); + }); + } + + for (const {STANDALONE, name} of noSPVcombinations) { + describe(`Deadlock (${name} Integration)`, function() { + this.timeout(10000); + const nodes = new NodesContext(regtest, 1); + let minerCtx; + let nodeCtx, address, node, wdb; + + before(async () => { + nodes.init({ + memory: false, + wallet: false + }); + + nodes.addNode({ + memory: false, + wallet: true, + standalone: STANDALONE + }); + + await nodes.open(); + + minerCtx = nodes.context(0); + nodeCtx = nodes.context(1); node = nodeCtx.node; wdb = nodeCtx.wdb; - await nodeCtx.open(); address = await wdb.primary.receiveAddress(); }); after(async () => { - await nodeCtx.close(); + await nodes.close(); }); - it('should generate 10 blocks', async () => { - await node.rpc.generateToAddress([10, address.toString(network)]); + it('should generate 20 blocks', async () => { + const BLOCKS = 20; + const chainBlocks = forEventCondition(node.chain, 'connect', (entry) => { + return entry.height === BLOCKS; + }, 5000); + + const wdbBlocks = forEventCondition(wdb, 'block connect', (entry) => { + return entry.height === BLOCKS; + }, 5000); + + await minerCtx.mineBlocks(BLOCKS, address); + await Promise.all([ + chainBlocks, + wdbBlocks + ]); }); it('should rescan when receiving a block', async () => { const preTip = await wdb.getTip(); + const blocks = forEventCondition(node.chain, 'connect', (entry) => { + return entry.height === preTip.height + 5; + }); + const wdbBlocks = forEventCondition(wdb, 'block connect', (entry) => { + return entry.height === preTip.height + 5; + }); await Promise.all([ - node.rpc.generateToAddress([1, address.toString(network)]), + minerCtx.mineBlocks(5, address), wdb.rescan(0) ]); + await blocks; + await wdbBlocks; + const wdbTip = await wdb.getTip(); - assert.strictEqual(wdbTip.height, preTip.height + 1); + assert.strictEqual(wdbTip.height, preTip.height + 5); }); - it('should rescan when receiving a block', async () => { + it('should rescan when receiving blocks', async () => { const preTip = await wdb.getTip(); + const minerHeight = minerCtx.height; + const BLOCKS = 50; - await Promise.all([ - wdb.rescan(0), - node.rpc.generateToAddress([1, address.toString(network)]) - ]); + const blocks = forEventCondition(node.chain, 'connect', (entry) => { + return entry.height === minerHeight + BLOCKS; + }); + + const wdbBlocks = forEventCondition(wdb, 'block connect', (entry) => { + return entry.height === minerHeight + BLOCKS; + }); + + const promises = [ + minerCtx.mineBlocks(BLOCKS, address) + ]; + + await forEvent(node.chain, 'connect'); + promises.push(wdb.rescan(0)); + await Promise.all(promises); + + await blocks; + await wdbBlocks; const tip = await wdb.getTip(); - assert.strictEqual(tip.height, preTip.height + 1); + + assert.strictEqual(tip.height, preTip.height + BLOCKS); + }); + + it('should rescan when chain is reorging', async () => { + const minerHeight = minerCtx.height; + const BLOCKS = 50; + const reorgHeight = minerHeight - 10; + const newHeight = minerHeight + 40; + + const blocks = forEventCondition(node.chain, 'connect', (entry) => { + return entry.height === newHeight; + }, 10000); + + const walletBlocks = forEventCondition(wdb, 'block connect', (entry) => { + return entry.height === newHeight; + }, 10000); + + const reorgEntry = await minerCtx.chain.getEntry(reorgHeight); + + const promises = [ + minerCtx.mineBlocks(BLOCKS, address, reorgEntry) + ]; + + // We start rescan only after first disconnect is detected to ensure + // wallet guard is set. + await forEvent(node.chain, 'disconnect'); + promises.push(wdb.rescan(0)); + await Promise.all(promises); + + await blocks; + await walletBlocks; + + const tip = await wdb.getTip(); + assert.strictEqual(tip.height, newHeight); + }); + + // Rescanning alternate chain. + it('should rescan when chain is reorging (alternate chain)', async () => { + const minerHeight = minerCtx.height; + const BLOCKS = 50; + const reorgHeight = minerHeight - 20; + + const reorgEntry = await minerCtx.chain.getEntry(reorgHeight); + const mineBlocks = minerCtx.mineBlocks(BLOCKS, address, reorgEntry); + + // We start rescan only after first disconnect is detected to ensure + // wallet guard is set. + await forEvent(node.chain, 'disconnect'); + + // abort should also report reason as an error. + const errorEvents = forEvent(wdb, 'error', 1); + + let err; + try { + // Because we are rescanning within the rescan blocks, + // these blocks will end up in alternate chain, resulting + // in error. + await wdb.rescan(minerHeight - 5); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'Cannot rescan an alternate chain.'); + + const errors = await errorEvents; + assert.strictEqual(errors.length, 1); + const errEv = errors[0].values[0]; + assert(errEv); + assert.strictEqual(errEv.message, 'Cannot rescan an alternate chain.'); + + await mineBlocks; }); }); + } }); + +async function deriveAddresses(walletClient, depth) { + const accInfo = await walletClient.getAccount('default'); + let currentDepth = accInfo.receiveDepth; + + if (depth <= currentDepth) + return; + + while (currentDepth !== depth) { + const addr = await walletClient.createAddress('default'); + currentDepth = addr.index; + } +} + +async function getAddress(walletClient, depth = -1, network = regtest) { + const accInfo = await walletClient.getAccount('default'); + const {accountKey, lookahead} = accInfo; + + if (depth === -1) + depth = accInfo.receiveDepth; + + const XPUBKey = HDPublicKey.fromBase58(accountKey, network); + const key = XPUBKey.derive(0).derive(depth).publicKey; + const address = Address.fromPubkey(key); + + const gappedDepth = depth + lookahead + 1; + return {address, depth, gappedDepth}; +} + +async function generateGappedAddresses(walletClient, count, network = regtest) { + let depth = -1; + + const addresses = []; + + // generate gapped addresses. + for (let i = 0; i < count; i++) { + const addrInfo = await getAddress(walletClient, depth, network); + + addresses.push({ + address: addrInfo.address, + depth: addrInfo.depth, + gappedDepth: addrInfo.gappedDepth + }); + + depth = addrInfo.gappedDepth; + } + + return addresses; +} diff --git a/test/wallet-unit-test.js b/test/wallet-unit-test.js index ce7749a73..1e6a55302 100644 --- a/test/wallet-unit-test.js +++ b/test/wallet-unit-test.js @@ -5,20 +5,24 @@ const blake2b = require('bcrypto/lib/blake2b'); const base58 = require('bcrypto/lib/encoding/base58'); const random = require('bcrypto/lib/random'); const bio = require('bufio'); -const { - HDPrivateKey, - Mnemonic, - WalletDB, - Network, - wallet: { Wallet } -} = require('../lib/hsd'); +const Network = require('../lib/protocol/network'); +const MTX = require('../lib/primitives/mtx'); +const HDPrivateKey = require('../lib/hd/private'); +const Mnemonic = require('../lib/hd/mnemonic'); +const WalletDB = require('../lib/wallet/walletdb'); +const Wallet = require('../lib/wallet/wallet'); const Account = require('../lib/wallet/account'); +const wutils = require('./util/wallet'); +const {nextEntry, fakeEntry} = require('./util/wallet'); +const MemWallet = require('./util/memwallet'); + +/** @typedef {import('../lib/primitives/tx')} TX */ const mnemonics = require('./data/mnemonic-english.json'); const network = Network.get('main'); describe('Wallet Unit Tests', () => { - describe('constructor', () => { + describe('constructor', function() { // abandon, abandon... about const phrase = mnemonics[0][1]; const passphrase = mnemonics[0][2]; @@ -346,4 +350,211 @@ describe('Wallet Unit Tests', () => { } }); }); + + describe('addBlock', function() { + const ALT_SEED = 0xdeadbeef; + + /** @type {WalletDB} */ + let wdb; + /** @type {Wallet} */ + let wallet; + /** @type {MemWallet} */ + let memwallet; + + beforeEach(async () => { + wdb = new WalletDB({ + network: network.type, + memory: true + }); + + await wdb.open(); + wallet = wdb.primary; + + memwallet = new MemWallet({ + network + }); + + for (let i = 0; i < 10; i++) { + const entry = nextEntry(wdb); + await wdb.addBlock(entry, []); + } + }); + + afterEach(async () => { + await wdb.close(); + wdb = null; + }); + + // Move forward + it('should progress with 10 block', async () => { + const tip = await wdb.getTip(); + + for (let i = 0; i < 10; i++) { + const entry = nextEntry(wdb); + const added = await wdb.addBlock(entry, []); + assert.ok(added); + assert.strictEqual(added.txs, 0); + assert.strictEqual(added.filterUpdated, false); + assert.equal(wdb.height, entry.height); + } + + assert.strictEqual(wdb.height, tip.height + 10); + }); + + it('should return number of transactions added (owned)', async () => { + const tip = await wdb.getTip(); + const wtx = await fakeWTX(wallet); + const entry = nextEntry(wdb); + const added = await wdb.addBlock(entry, [wtx]); + + assert.ok(added); + assert.strictEqual(added.txs, 1); + assert.strictEqual(added.filterUpdated, true); + assert.equal(wdb.height, tip.height + 1); + }); + + it('should return number of transactions added (none)', async () => { + const tip = await wdb.getTip(); + const entry = nextEntry(wdb); + const added = await wdb.addBlock(entry, []); + + assert.ok(added); + assert.strictEqual(added.txs, 0); + assert.strictEqual(added.filterUpdated, false); + assert.equal(wdb.height, tip.height + 1); + }); + + it('should fail to add block on unusual reorg', async () => { + const tip = await wdb.getTip(); + const entry = nextEntry(wdb, ALT_SEED, ALT_SEED); + + // TODO: Detect sync chain is correct. + const added = await wdb.addBlock(entry, []); + assert.strictEqual(added, null); + assert.strictEqual(wdb.height, tip.height); + }); + + // Same block + it('should re-add the same block', async () => { + const tip = await wdb.getTip(); + const entry = nextEntry(wdb); + const wtx1 = await fakeWTX(wallet); + const wtx2 = await fakeWTX(wallet); + + const added1 = await wdb.addBlock(entry, [wtx1]); + assert.ok(added1); + assert.strictEqual(added1.txs, 1); + assert.strictEqual(added1.filterUpdated, true); + assert.equal(wdb.height, tip.height + 1); + + // Same TX wont show up second time. + const added2 = await wdb.addBlock(entry, [wtx1]); + assert.ok(added2); + assert.strictEqual(added2.txs, 0); + assert.strictEqual(added2.filterUpdated, false); + assert.equal(wdb.height, tip.height + 1); + + const added3 = await wdb.addBlock(entry, [wtx1, wtx2]); + assert.ok(added3); + assert.strictEqual(added3.txs, 1); + // Both txs are using the same address. + assert.strictEqual(added3.filterUpdated, false); + assert.equal(wdb.height, tip.height + 1); + }); + + it('should ignore txs not owned by wallet', async () => { + const tip = await wdb.getTip(); + const addr = memwallet.getReceive().toString(network); + const tx = fakeTX(addr); + + const entry = nextEntry(wdb); + const added = await wdb.addBlock(entry, [tx]); + assert.ok(added); + assert.strictEqual(added.txs, 0); + assert.strictEqual(added.filterUpdated, false); + + assert.strictEqual(wdb.height, tip.height + 1); + }); + + // This should not happen, but there should be guards in place. + it('should resync if the block is the same', async () => { + const tip = await wdb.getTip(); + const entry = fakeEntry(tip.height, 0, ALT_SEED); + + // TODO: Detect sync chain is correct. + const added = await wdb.addBlock(entry, []); + assert.strictEqual(added, null); + }); + + // LOW BLOCKS + it('should ignore blocks before tip', async () => { + const tip = await wdb.getTip(); + const entry = fakeEntry(tip.height - 1); + const wtx = await fakeWTX(wallet); + + // ignore low blocks. + const added = await wdb.addBlock(entry, [wtx]); + assert.strictEqual(added, null); + assert.strictEqual(wdb.height, tip.height); + }); + + it('should sync chain blocks before tip on unusual low block reorg', async () => { + const tip = await wdb.getTip(); + const entry = fakeEntry(tip.height - 1, 0, ALT_SEED); + const wtx = await fakeWTX(wallet); + + // TODO: Detect sync chain is correct. + + // ignore low blocks. + const added = await wdb.addBlock(entry, [wtx]); + assert.strictEqual(added, null); + assert.strictEqual(wdb.height, tip.height); + }); + + // HIGH BLOCKS + it('should rescan for missed blocks', async () => { + const tip = await wdb.getTip(); + // next + 1 + const entry = fakeEntry(tip.height + 2); + + let rescan = false; + let rescanHash = null; + + wdb.client.rescanInteractive = async (hash) => { + rescan = true; + rescanHash = hash; + }; + + const added = await wdb.addBlock(entry, []); + assert.strictEqual(added, null); + + assert.strictEqual(rescan, true); + assert.bufferEqual(rescanHash, tip.hash); + }); + }); }); + +/** + * @param {String} addr + * @returns {TX} + */ + +function fakeTX(addr) { + const tx = new MTX(); + tx.addInput(wutils.dummyInput()); + tx.addOutput({ + address: addr, + value: 5460 + }); + return tx.toTX(); +} + +/** + * @param {Wallet} wallet + * @returns {Promise} + */ + +async function fakeWTX(wallet) { + const addr = await wallet.receiveAddress(); + return fakeTX(addr); +}