From 6f8a2573a244a61159b16510ac44e403a2a0ae18 Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Sun, 13 Jan 2019 09:11:43 -0500 Subject: [PATCH 01/46] [1.0.0.0] Adopt std thread (#108) * add thread_group to replace boost::thread_group * move src from boost::thread to std::thread --- configure.ac | 51 --------- src/.formatted-files | 6 +- src/Makefile.am | 5 +- src/Makefile.test.include | 3 +- src/args.cpp | 1 - src/blockgeneration/blockgeneration.cpp | 12 +- src/blockgeneration/blockgeneration.h | 6 +- src/blockgeneration/miner.cpp | 8 +- src/blockgeneration/minter.cpp | 8 +- src/chain/chainman.cpp | 15 ++- src/checkqueue.h | 19 +-- src/eccoind.cpp | 15 +-- src/httpserver.cpp | 30 ++--- src/httpserver.h | 2 +- src/init.cpp | 87 ++++++++------ src/init.h | 5 +- src/main.cpp | 2 +- src/messages.cpp | 6 +- src/net.cpp | 112 ++++++++++++------ src/net.h | 12 +- src/netbase.cpp | 2 +- src/processblock.cpp | 15 ++- src/processblock.h | 1 + src/rpc/rpcblockchain.cpp | 8 +- src/scheduler.cpp | 146 ------------------------ src/scheduler.h | 98 ---------------- src/sync.cpp | 115 +++++++++++++------ src/test/scheduler_tests.cpp | 127 --------------------- src/test/test_bitcoin.cpp | 1 - src/test/test_bitcoin.h | 1 + src/test/util_tests.cpp | 6 +- src/threadgroup.h | 42 +++++++ src/threadinterrupt.cpp | 48 -------- src/threadinterrupt.h | 50 -------- src/torcontrol.cpp | 6 +- src/torcontrol.h | 4 +- src/txdb.cpp | 13 ++- src/util/util.cpp | 11 +- src/util/utiltime.cpp | 6 +- src/verifydb.cpp | 15 ++- src/wallet/db.cpp | 5 + src/wallet/wallet.cpp | 4 +- src/wallet/walletdb.cpp | 15 ++- 43 files changed, 396 insertions(+), 748 deletions(-) delete mode 100644 src/scheduler.cpp delete mode 100644 src/scheduler.h delete mode 100644 src/test/scheduler_tests.cpp create mode 100644 src/threadgroup.h delete mode 100644 src/threadinterrupt.cpp delete mode 100644 src/threadinterrupt.h diff --git a/configure.ac b/configure.ac index 5d2a639f..fb769157 100644 --- a/configure.ac +++ b/configure.ac @@ -781,57 +781,6 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" -dnl Boost >= 1.50 uses sleep_for rather than the now-deprecated sleep, however -dnl it was broken from 1.50 to 1.52 when backed by nanosleep. Use sleep_for if -dnl a working version is available, else fall back to sleep. sleep was removed -dnl after 1.56. -dnl If neither is available, abort. -TEMP_LIBS="$LIBS" -LIBS="$BOOST_LIBS $LIBS" -TEMP_CPPFLAGS="$CPPFLAGS" -CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" -AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #include - #include - ]],[[ - #if BOOST_VERSION >= 105000 && (!defined(BOOST_HAS_NANOSLEEP) || BOOST_VERSION >= 105200) - boost::this_thread::sleep_for(boost::chrono::milliseconds(0)); - #else - choke me - #endif - ]])], - [boost_sleep=yes; - AC_DEFINE(HAVE_WORKING_BOOST_SLEEP_FOR, 1, [Define this symbol if boost sleep_for works])], - [boost_sleep=no]) -LIBS="$TEMP_LIBS" -CPPFLAGS="$TEMP_CPPFLAGS" - -if test x$boost_sleep != xyes; then -TEMP_LIBS="$LIBS" -LIBS="$BOOST_LIBS $LIBS" -TEMP_CPPFLAGS="$CPPFLAGS" -CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" -AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #include - #include - #include - ]],[[ - #if BOOST_VERSION <= 105600 - boost::this_thread::sleep(boost::posix_time::milliseconds(0)); - #else - choke me - #endif - ]])], - [boost_sleep=yes; AC_DEFINE(HAVE_WORKING_BOOST_SLEEP, 1, [Define this symbol if boost sleep works])], - [boost_sleep=no]) -LIBS="$TEMP_LIBS" -CPPFLAGS="$TEMP_CPPFLAGS" -fi - -if test x$boost_sleep != xyes; then - AC_MSG_ERROR(No working boost sleep implementation found.) -fi - fi if test x$use_pkgconfig = xyes; then diff --git a/src/.formatted-files b/src/.formatted-files index c89f8e34..7bdec713 100644 --- a/src/.formatted-files +++ b/src/.formatted-files @@ -133,8 +133,6 @@ rpc/rpcrawtransaction.cpp rpc/rpcserver.cpp rpc/rpcserver.h rpc/rpcwallet.cpp -scheduler.cpp -scheduler.h script/bitcoinconsensus.cpp script/bitcoinconsensus.h script/interpreter.cpp @@ -161,8 +159,7 @@ support/pagelocker.cpp support/pagelocker.h sync.cpp sync.h -threadinterrupt.cpp -threadinterrupt.h +threadgroup.h threadsafety.h timedata.cpp timedata.h @@ -235,7 +232,6 @@ test/prevector_tests.cpp test/reverselock_tests.cpp test/rpc_tests.cpp test/sanity_tests.cpp -test/scheduler_tests.cpp test/scriptnum10.h test/scriptnum_tests.cpp test/script_P2SH_tests.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 39b9c37a..bc19f550 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -110,7 +110,6 @@ BITCOIN_CORE_H = \ rpc/rpcclient.h \ rpc/rpcprotocol.h \ rpc/rpcserver.h \ - scheduler.h \ script/interpreter.h \ script/script.h \ script/script_error.h \ @@ -126,7 +125,7 @@ BITCOIN_CORE_H = \ support/cleanse.h \ support/pagelocker.h \ sync.h \ - threadinterrupt.h \ + threadgroup.h \ threadsafety.h \ timedata.h \ tinyformat.h \ @@ -218,7 +217,6 @@ libbitcoin_server_a_SOURCES = \ chain/tx.cpp \ protocol.cpp \ pubkey.cpp \ - scheduler.cpp \ pbkdf2.cpp \ script/interpreter.cpp \ script/script.cpp \ @@ -271,7 +269,6 @@ libbitcoin_server_a_SOURCES = \ wallet/wallet_ismine.cpp \ wallet/walletdb.cpp \ rpc/rpcclient.cpp \ - threadinterrupt.cpp \ $(BITCOIN_CORE_H) if GLIBC_BACK_COMPAT diff --git a/src/Makefile.test.include b/src/Makefile.test.include index db52e257..5ed73237 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -85,7 +85,6 @@ BITCOIN_TESTS = \ test/pmt_tests.cpp \ test/reverselock_tests.cpp \ test/sanity_tests.cpp \ - test/scheduler_tests.cpp \ test/scriptnum_tests.cpp \ test/serialize_tests.cpp \ test/sigopcount_tests.cpp \ @@ -96,7 +95,7 @@ BITCOIN_TESTS = \ test/univalue_tests.cpp BITCOIN_TESTS += \ - wallet/test/wallet_tests.cpp + wallet/test/wallet_tests.cpp # wallet/test/walletdb_tests.cpp \ # wallet/test/crypto_tests.cpp diff --git a/src/args.cpp b/src/args.cpp index 2dd9323f..80c888e1 100644 --- a/src/args.cpp +++ b/src/args.cpp @@ -90,7 +90,6 @@ #include // for to_lower() #include // for startswith() and endswith() #include -#include #include #include #include diff --git a/src/blockgeneration/blockgeneration.cpp b/src/blockgeneration/blockgeneration.cpp index d09de692..67f02556 100644 --- a/src/blockgeneration/blockgeneration.cpp +++ b/src/blockgeneration/blockgeneration.cpp @@ -67,7 +67,7 @@ std::unique_ptr CreateNewBlock(CWallet *pwallet, const CScript & return CreateNewPoWBlock(pwallet, scriptPubKeyIn); } -boost::thread_group *minerThreads = nullptr; +thread_group *minerThreads = nullptr; void ThreadMiner(void *parg, bool shutdownOnly) { @@ -82,11 +82,11 @@ void ThreadMiner(void *parg, bool shutdownOnly) { return; } - minerThreads = new boost::thread_group(); + minerThreads = new thread_group(); CWallet *pwallet = (CWallet *)parg; try { - minerThreads->create_thread(boost::bind(&EccMiner, pwallet)); + minerThreads->create_thread("eccMiner", &EccMiner, pwallet); } catch (std::exception &e) { @@ -101,7 +101,7 @@ void ThreadMiner(void *parg, bool shutdownOnly) LogPrintf("Thread Miner thread exiting \n"); } -boost::thread_group *minterThreads = nullptr; +thread_group *minterThreads = nullptr; void ThreadMinter(void *parg, bool shutdownOnly) { @@ -116,11 +116,11 @@ void ThreadMinter(void *parg, bool shutdownOnly) { return; } - minterThreads = new boost::thread_group(); + minterThreads = new thread_group(); CWallet *pwallet = (CWallet *)parg; try { - minterThreads->create_thread(boost::bind(&EccMinter, pwallet)); + minterThreads->create_thread("eccMinter", &EccMinter, pwallet); } catch (std::exception &e) { diff --git a/src/blockgeneration/blockgeneration.h b/src/blockgeneration/blockgeneration.h index bdc5a94b..7f0821e9 100644 --- a/src/blockgeneration/blockgeneration.h +++ b/src/blockgeneration/blockgeneration.h @@ -22,8 +22,6 @@ #include "consensus/params.h" #include "wallet/wallet.h" -#include - #ifndef ECCOIN_BLOCKGENERATION_H #define ECCOIN_BLOCKGENERATION_H @@ -45,7 +43,7 @@ std::unique_ptr CreateNewBlock(CWallet *pwallet, const CScript & void ThreadGeneration(void *parg, bool shutdownOnly = false, bool fProofOfStake = false); -extern boost::thread_group *minerThreads; -extern boost::thread_group *minterThreads; +extern thread_group *minerThreads; +extern thread_group *minterThreads; #endif // ECCOIN_BLOCKGENERATION_H diff --git a/src/blockgeneration/miner.cpp b/src/blockgeneration/miner.cpp index 045ec9a1..b5a3882e 100644 --- a/src/blockgeneration/miner.cpp +++ b/src/blockgeneration/miner.cpp @@ -442,19 +442,19 @@ void EccMiner(CWallet *pwallet) unsigned int nExtraNonce = 0; while (true) { - if (fShutdown) + if (shutdown_threads.load()) return; if (!g_connman) { MilliSleep(1000); - if (fShutdown) + if (shutdown_threads.load()) return; } while (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) < 6 || pnetMan->getChainActive()->IsInitialBlockDownload() || pwallet->IsLocked()) { MilliSleep(1000); - if (fShutdown) + if (shutdown_threads.load()) return; } // @@ -543,7 +543,7 @@ void EccMiner(CWallet *pwallet) } } // Check for stop or if block needs to be rebuilt - if (fShutdown) + if (shutdown_threads.load()) return; if (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL)) break; diff --git a/src/blockgeneration/minter.cpp b/src/blockgeneration/minter.cpp index 070ed775..a0d0fc80 100644 --- a/src/blockgeneration/minter.cpp +++ b/src/blockgeneration/minter.cpp @@ -159,7 +159,7 @@ std::unique_ptr CreateNewPoSBlock(CWallet *pwallet, const CScrip nLastCoinStakeSearchTime = nSearchTime; } MilliSleep(100); - if (fShutdown) + if (shutdown_threads.load()) return nullptr; } @@ -352,19 +352,19 @@ void EccMinter(CWallet *pwallet) unsigned int nExtraNonce = 0; while (true) { - if (fShutdown) + if (shutdown_threads.load()) return; if (!g_connman) { MilliSleep(1000); - if (fShutdown) + if (shutdown_threads.load()) return; } while (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) < 6 || pnetMan->getChainActive()->IsInitialBlockDownload() || pwallet->IsLocked()) { MilliSleep(1000); - if (fShutdown) + if (shutdown_threads.load()) return; } CBlockIndex *pindexPrev = pnetMan->getChainActive()->chainActive.Tip(); diff --git a/src/chain/chainman.cpp b/src/chain/chainman.cpp index 5f2f88cb..a42a1f50 100644 --- a/src/chain/chainman.cpp +++ b/src/chain/chainman.cpp @@ -28,7 +28,6 @@ #include "processheader.h" #include "txmempool.h" #include "undo.h" -#include CBlockIndex *CChainManager::AddToBlockIndex(const CBlockHeader &block) { @@ -189,10 +188,17 @@ bool CChainManager::LoadBlockIndexDB() { int64_t nStart = GetTimeMillis(); if (!pblocktree->LoadBlockIndexGuts()) + { return false; + } + LogPrintf("LoadBlockIndexGuts %15dms\n", GetTimeMillis() - nStart); - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + LogPrintf("LoadBlockIndexDB(): Shutdown requested. returning...\n"); + return false; + } // Calculate nChainWork std::vector > vSortedByHeight; @@ -324,7 +330,10 @@ bool CChainManager::LoadExternalBlockFile(const CNetworkTemplate &chainparams, F uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + break; + } blkdat.SetPos(nRewind); nRewind++; // start one byte further next time, in case of failure diff --git a/src/checkqueue.h b/src/checkqueue.h index 6674d93a..aea7d0d4 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -27,6 +27,8 @@ #include #include +extern std::atomic shutdown_threads; + template class CCheckQueueControl; @@ -73,9 +75,6 @@ class CCheckQueue */ unsigned int nTodo; - //! Whether we're shutting down. - bool fQuit; - //! The maximum number of elements to be processed in one batch unsigned int nBatchSize; @@ -108,7 +107,7 @@ class CCheckQueue // logically, the do loop starts here while (queue.empty()) { - if ((fMaster || fQuit) && nTodo == 0) + if (fMaster && nTodo == 0) { nTotal--; bool fRet = fAllOk; @@ -121,6 +120,10 @@ class CCheckQueue nIdle++; cond.wait(lock); // wait nIdle--; + if (shutdown_threads.load() == true) + { + return true; + } } // Decide how many work units to process now. // * Do not try to do everything at once, but aim for increasingly smaller batches so @@ -144,16 +147,13 @@ class CCheckQueue if (fOk) fOk = check(); vChecks.clear(); + } while (true); } public: //! Create a new check queue - CCheckQueue(unsigned int nBatchSizeIn) - : nIdle(0), nTotal(0), fAllOk(true), nTodo(0), fQuit(false), nBatchSize(nBatchSizeIn) - { - } - + CCheckQueue(unsigned int nBatchSizeIn) : nIdle(0), nTotal(0), fAllOk(true), nTodo(0), nBatchSize(nBatchSizeIn) {} //! Worker thread void Thread() { Loop(); } //! Wait until execution finishes, and return whether all evaluations were successful. @@ -174,6 +174,7 @@ class CCheckQueue condWorker.notify_all(); } + void Stop() { condWorker.notify_all(); } ~CCheckQueue() {} bool IsIdle() { diff --git a/src/eccoind.cpp b/src/eccoind.cpp index 7923c858..9d6b6ba4 100644 --- a/src/eccoind.cpp +++ b/src/eccoind.cpp @@ -25,20 +25,19 @@ #include "networks/netman.h" #include "noui.h" #include "rpc/rpcserver.h" -#include "scheduler.h" +#include "threadgroup.h" #include "util/util.h" #include #include -#include #include -void WaitForShutdown(boost::thread_group *threadGroup) +void WaitForShutdown(thread_group *threadGroup) { bool fShutdown = ShutdownRequested(); // Tell the main threads to shutdown. - while (!fShutdown) + while (fShutdown == false) { MilliSleep(200); fShutdown = ShutdownRequested(); @@ -56,8 +55,7 @@ void WaitForShutdown(boost::thread_group *threadGroup) // bool AppInit(int argc, char *argv[]) { - boost::thread_group threadGroup; - CScheduler scheduler; + thread_group threadGroup; bool fRet = false; @@ -159,7 +157,7 @@ bool AppInit(int argc, char *argv[]) // Set this early so that parameter interactions go to console InitLogging(); InitParameterInteraction(); - fRet = AppInit2(threadGroup, scheduler); + fRet = AppInit2(threadGroup); } catch (const std::exception &e) { @@ -173,9 +171,6 @@ bool AppInit(int argc, char *argv[]) if (!fRet) { Interrupt(threadGroup); - // threadGroup.join_all(); was left out intentionally here, because we didn't re-test all of - // the startup-failure cases to make sure they don't result in a hang due to some - // thread-blocking-waiting-for-another-thread-during-startup case } else { diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 92d9ea36..99e18db9 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -490,17 +490,20 @@ bool InitHTTPServer() return true; } -boost::thread threadHTTP; +std::thread *threadHTTP; bool StartHTTPServer() { LogPrint("http", "Starting HTTP server\n"); int rpcThreads = std::max((long)gArgs.GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); - threadHTTP = boost::thread(boost::bind(&ThreadHTTP, eventBase, eventHTTP)); + threadHTTP = new std::thread(&ThreadHTTP, eventBase, eventHTTP); for (int i = 0; i < rpcThreads; i++) - boost::thread(boost::bind(&HTTPWorkQueueRun, workQueue)); + { + std::thread worker(&HTTPWorkQueueRun, workQueue); + worker.detach(); + } return true; } @@ -532,24 +535,9 @@ void StopHTTPServer() } if (eventBase) { - LogPrint("http", "Waiting for HTTP event thread to exit\n"); -// Give event loop a few seconds to exit (to send back last RPC responses), then break it -// Before this was solved with event_base_loopexit, but that didn't work as expected in -// at least libevent 2.0.21 and always introduced a delay. In libevent -// master that appears to be solved, so in the future that solution -// could be used again (if desirable). -// (see discussion in https://github.com/bitcoin/bitcoin/pull/6990) -#if BOOST_VERSION >= 105000 - if (!threadHTTP.try_join_for(boost::chrono::milliseconds(2000))) - { -#else - if (!threadHTTP.timed_join(boost::posix_time::milliseconds(2000))) - { -#endif - LogPrintf("HTTP event loop did not exit within allotted time, sending loopbreak\n"); - event_base_loopbreak(eventBase); - threadHTTP.join(); - } + LogPrint("http", "HTTP event thread exiting...\n"); + event_base_loopbreak(eventBase); + threadHTTP->join(); } if (eventHTTP) { diff --git a/src/httpserver.h b/src/httpserver.h index a85ce9b9..fb976ea0 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -22,7 +22,7 @@ #include #include -#include + #include #include diff --git a/src/init.cpp b/src/init.cpp index eaf4a597..86934e30 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -39,7 +39,6 @@ #include "policy/policy.h" #include "processblock.h" #include "rpc/rpcserver.h" -#include "scheduler.h" #include "script/sigcache.h" #include "script/standard.h" #include "torcontrol.h" @@ -71,10 +70,9 @@ #include #include #include -#include + #include -bool fShutdown = false; CWallet *pwalletMain = nullptr; CNetworkManager *pnetMan = nullptr; @@ -120,7 +118,7 @@ CClientUIInterface uiInterface; // Declared but not defined in ui_interface.h // created by AppInit() or the Qt main() function. // // A clean exit happens when StartShutdown() or the SIGTERM -// signal handler sets fRequestShutdown, which triggers +// signal handler sets shutdown_threads, which triggers // the DetectShutdownThread(), which interrupts the main thread group. // DetectShutdownThread() then exits, which causes AppInit() to // continue (it .joins the shutdown thread). @@ -133,15 +131,22 @@ CClientUIInterface uiInterface; // Declared but not defined in ui_interface.h // before adding any threads to the threadGroup, so .join_all() returns // immediately and the parent exits from main(). // -// Shutdown for Qt is very similar, only it uses a QTimer to detect -// fRequestShutdown getting set, and then does the normal Qt -// shutdown thing. // -volatile bool fRequestShutdown = false; -void StartShutdown() { fRequestShutdown = true; } -bool ShutdownRequested() { return fRequestShutdown; } +std::atomic shutdown_threads(false); + +void StartShutdown() +{ + // use one atomic bool for shutdowns + shutdown_threads.store(true); +} +bool ShutdownRequested() +{ + // use one atomic bool for shutdowns + return shutdown_threads.load(); +} + class CCoinsViewErrorCatcher final : public CCoinsViewBacked { public: @@ -171,15 +176,16 @@ std::unique_ptr pcoinsdbview; std::unique_ptr pcoinscatcher; static boost::scoped_ptr globalVerifyHandle; -void Interrupt(boost::thread_group &threadGroup) +void Interrupt(thread_group &threadGroup) { - fShutdown = true; + threadGroup.interrupt_all(); + g_connman->Interrupt(); InterruptHTTPServer(); InterruptHTTPRPC(); InterruptRPC(); InterruptREST(); InterruptTorControl(); - threadGroup.interrupt_all(); + InterruptScriptCheck(); } void Shutdown() @@ -188,7 +194,9 @@ void Shutdown() static CCriticalSection cs_Shutdown; TRY_LOCK(cs_Shutdown, lockShutdown); if (!lockShutdown) + { return; + } /// Note: Shutdown() must be able to handle cases in which AppInit2() failed part of the way, /// for example if the data directory was found to be locked. @@ -204,15 +212,20 @@ void Shutdown() if (pwalletMain) pwalletMain->Flush(false); + // shut off pos miner ThreadGeneration(pwalletMain, true, true); // shut off pow miner ThreadGeneration(pwalletMain, true, false); + MapPort(false); UnregisterValidationInterface(peerLogic.get()); peerLogic.reset(); + g_connman.reset(); + StopTorControl(); + UnregisterNodeSignals(GetNodeSignals()); if (fFeeEstimatesInitialized) @@ -230,25 +243,28 @@ void Shutdown() LOCK(cs_main); if (pnetMan) { - if (pnetMan->getChainActive()->pcoinsTip != NULL) + if (pnetMan->getChainActive()->pcoinsTip != nullptr) { FlushStateToDisk(); } pnetMan->getChainActive()->pcoinsTip.reset(); + pnetMan->getChainActive()->pcoinsTip = nullptr; } pcoinscatcher.reset(); - pcoinscatcher = NULL; + pcoinscatcher = nullptr; pcoinsdbview.reset(); - pcoinsdbview = NULL; + pcoinsdbview = nullptr; if (pnetMan) { pnetMan->getChainActive()->pblocktree.reset(); - pnetMan->getChainActive()->pblocktree = NULL; + pnetMan->getChainActive()->pblocktree = nullptr; } } if (pwalletMain) + { pwalletMain->Flush(true); + } #ifndef WIN32 try @@ -272,7 +288,7 @@ void Shutdown() /** * Signal handlers are very limited in what they are allowed to do, so: */ -void HandleSIGTERM(int) { fRequestShutdown = true; } +void HandleSIGTERM(int) { shutdown_threads.store(true); } void HandleSIGHUP(int) { fReopenDebugLog = true; } bool static InitError(const std::string &str) { @@ -666,7 +682,7 @@ static void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex std::string strCmd = gArgs.GetArg("-blocknotify", ""); boost::replace_all(strCmd, "%s", pBlockIndex->GetBlockHash().GetHex()); - boost::thread t(runCommand, strCmd); // thread runs free + std::thread t(runCommand, strCmd); // thread runs free } struct CImportingNow @@ -771,7 +787,7 @@ bool InitSanityCheck(void) return true; } -bool AppInitServers(boost::thread_group &threadGroup) +bool AppInitServers(thread_group &threadGroup) { RPCServer::OnStopped(&OnRPCStopped); RPCServer::OnPreCommand(&OnRPCPreCommand); @@ -902,7 +918,7 @@ void GenerateNetworkTemplates() /** Initialize bitcoin. * @pre Parameters should be parsed and config file should be read. */ -bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler) +bool AppInit2(thread_group &threadGroup) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -1216,13 +1232,12 @@ bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler) if (nScriptCheckThreads) { for (int i = 0; i < nScriptCheckThreads - 1; i++) - threadGroup.create_thread(&ThreadScriptCheck); + { + std::string name = "scriptCheck" + std::to_string(i); + threadGroup.create_thread(name, &ThreadScriptCheck); + } } - // Start the lightweight task scheduler thread - CScheduler::Function serviceLoop = boost::bind(&CScheduler::serviceQueue, &scheduler); - threadGroup.create_thread(boost::bind(&TraceThread, "scheduler", serviceLoop)); - /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections * that the server is there and will be ready later). Warmup mode will @@ -1606,7 +1621,7 @@ bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler) if (fRet) { fReindex = true; - fRequestShutdown = false; + shutdown_threads.store(false); } else { @@ -1624,7 +1639,7 @@ bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler) // As LoadBlockIndex can take several minutes, it's possible the user // requested to kill the GUI during the last operation. If so, exit. // As the program has not fully started yet, Shutdown() is possibly overkill. - if (fRequestShutdown) + if (shutdown_threads.load()) { LogPrintf("Shutdown requested. Exiting.\n"); return false; @@ -1665,13 +1680,15 @@ bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler) for (auto const &strFile : gArgs.GetArgs("-loadblock")) vImportFiles.push_back(strFile); } - threadGroup.create_thread(boost::bind(&ThreadImport, vImportFiles)); - /// THIS FOR LOOP IS STUCK FOREVER BECAUSE CHAIN TIP IS NEVER ACTIVATED + threadGroup.create_thread("importFiles", &ThreadImport, vImportFiles); + if (pnetMan->getChainActive()->chainActive.Tip() == NULL) { LogPrintf("Waiting for genesis block to be imported...\n"); - while (!fRequestShutdown && pnetMan->getChainActive()->chainActive.Tip() == NULL) + while (!shutdown_threads.load() && pnetMan->getChainActive()->chainActive.Tip() == NULL) + { MilliSleep(10); + } } // ********************************************************* Step 11: start node @@ -1693,7 +1710,7 @@ bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler) LogPrintf("mapAddressBook.size() = %u\n", pwalletMain ? pwalletMain->mapAddressBook.size() : 0); if (gArgs.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) - StartTorControl(threadGroup, scheduler); + StartTorControl(threadGroup); Discover(threadGroup); @@ -1716,7 +1733,7 @@ bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler) connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; connOptions.nMaxOutboundLimit = nMaxOutboundLimit; - if (!connman.Start(scheduler, strNodeError, connOptions)) + if (!connman.Start(strNodeError, connOptions)) { return InitError(strNodeError); } @@ -1743,8 +1760,8 @@ bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler) pwalletMain->ReacceptWalletTransactions(); // Run a thread to flush wallet periodically - threadGroup.create_thread(boost::bind(&ThreadFlushWalletDB, boost::ref(pwalletMain->strWalletFile))); + threadGroup.create_thread("flushWalletDB", &ThreadFlushWalletDB, boost::ref(pwalletMain->strWalletFile)); } - return !fRequestShutdown; + return !shutdown_threads.load(); } diff --git a/src/init.h b/src/init.h index 307a1c2e..dbe47d74 100644 --- a/src/init.h +++ b/src/init.h @@ -40,21 +40,20 @@ extern CNetworkManager *pnetMan; void StartShutdown(); bool ShutdownRequested(); /** Interrupt threads */ -void Interrupt(boost::thread_group &threadGroup); +void Interrupt(thread_group &threadGroup); void Shutdown(); //! Initialize the logging infrastructure void InitLogging(); //! Parameter interaction: change current parameters depending on various rules void InitParameterInteraction(); void GenerateNetworkTemplates(); -bool AppInit2(boost::thread_group &threadGroup, CScheduler &scheduler); +bool AppInit2(thread_group &threadGroup); /** Help for options shared between UI and daemon (for -help) */ std::string HelpMessage(); /** Returns licensing information (for -version) */ std::string LicenseInfo(); -extern bool fShutdown; extern std::unique_ptr pcoinsdbview; diff --git a/src/main.cpp b/src/main.cpp index ba0b966a..7302f6dc 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -62,7 +62,7 @@ #include #include #include -#include + #include #include #include diff --git a/src/messages.cpp b/src/messages.cpp index f784a88d..326f34b0 100644 --- a/src/messages.cpp +++ b/src/messages.cpp @@ -46,13 +46,11 @@ #include "version.h" #include -#include -#include -#include #include -#include #include +#include + ////////////////////////////////////////////////////////////////////////////// // // Messages diff --git a/src/net.cpp b/src/net.cpp index a6a3428c..f01f4daf 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -29,7 +29,6 @@ #include "crypto/hash.h" #include "init.h" #include "networks/netman.h" -#include "scheduler.h" #include "ui_interface.h" #include "util/utilstrencodings.h" @@ -47,7 +46,7 @@ #endif #include -#include + #include #include @@ -1288,7 +1287,7 @@ void CConnman::AcceptConnection(const ListenSocket &hListenSocket) void CConnman::ThreadSocketHandler() { unsigned int nPrevNodeCount = 0; - while (!interruptNet) + while (interruptNet.load() == false) { // // Disconnect nodes @@ -1428,7 +1427,7 @@ void CConnman::ThreadSocketHandler() } int nSelect = select(have_fds ? hSocketMax + 1 : 0, &fdsetRecv, &fdsetSend, &fdsetError, &timeout); - if (interruptNet) + if (interruptNet.load() == true) { return; } @@ -1446,7 +1445,8 @@ void CConnman::ThreadSocketHandler() } FD_ZERO(&fdsetSend); FD_ZERO(&fdsetError); - if (!interruptNet.sleep_for(std::chrono::milliseconds(timeout.tv_usec / 1000))) + MilliSleep(timeout.tv_usec / 1000); + if (interruptNet.load() == true) { return; } @@ -1477,7 +1477,7 @@ void CConnman::ThreadSocketHandler() } for (CNode *pnode : vNodesCopy) { - if (interruptNet) + if (interruptNet.load() == true) { return; } @@ -1632,6 +1632,8 @@ void CConnman::WakeMessageHandler() condMsgProc.notify_one(); } +std::atomic upnp_thread_shutdown(false); + #ifdef USE_UPNP void ThreadMapPort() { @@ -1712,9 +1714,21 @@ void ThreadMapPort() { LogPrintf("UPnP Port Mapping successful.\n"); } - + if (upnp_thread_shutdown.load()) + { + return; + } + int64_t nStart = GetTime(); + int64_t nFinish = nStart + (20 * 60 * 1000); // Refresh every 20 minutes - MilliSleep(20 * 60 * 1000); + while (nStart <= nFinish && upnp_thread_shutdown.load() == false) + { + MilliSleep(10000); + } + if (upnp_thread_shutdown.load()) + { + return; + } } } catch (const boost::thread_interrupted &) @@ -1741,21 +1755,22 @@ void ThreadMapPort() void MapPort(bool fUseUPnP) { - static boost::thread *upnp_thread = nullptr; + static std::thread *upnp_thread = nullptr; if (fUseUPnP) { if (upnp_thread) { - upnp_thread->interrupt(); + upnp_thread_shutdown.store(true); upnp_thread->join(); delete upnp_thread; + upnp_thread = nullptr; } - upnp_thread = new boost::thread(boost::bind(&TraceThread, "upnp", &ThreadMapPort)); + upnp_thread = new std::thread(&TraceThread, "upnp", &ThreadMapPort); } else if (upnp_thread) { - upnp_thread->interrupt(); + upnp_thread_shutdown.store(true); upnp_thread->join(); delete upnp_thread; upnp_thread = nullptr; @@ -1792,7 +1807,8 @@ void CConnman::ThreadDNSAddressSeed() // less influence on the network topology, and reduces traffic to the seeds. if ((addrman.size() > 0) && (!gArgs.GetBoolArg("-forcednsseed", DEFAULT_FORCEDNSSEED))) { - if (!interruptNet.sleep_for(std::chrono::seconds(11))) + MilliSleep(5000); + if (interruptNet.load() == true) { return; } @@ -1866,12 +1882,31 @@ void CConnman::DumpAddresses() LogPrintf("Flushed %d addresses to peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); } -void CConnman::DumpData() +void CConnman::_DumpData() { DumpAddresses(); DumpBanlist(); } +void CConnman::DumpData(int64_t seconds_between_runs) +{ + while (interruptNet.load() == false) + { + // this has the potential to be a long sleep. so do it in chunks incase of node shutdown + int64_t nStart = GetTime(); + int64_t nEnd = nStart + seconds_between_runs; + while (nStart < nEnd) + { + if (interruptNet.load() == true) + { + break; + } + std::this_thread::sleep_for(std::chrono::seconds(2)); + } + _DumpData(); + } +} + void CConnman::ProcessOneShot() { std::string strDest; @@ -1909,13 +1944,15 @@ void CConnman::ThreadOpenConnections() OpenNetworkConnection(addr, false, nullptr, strAddr.c_str()); for (int i = 0; i < 10 && i < nLoop; i++) { - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) + MilliSleep(500); + if (interruptNet.load() == true) { return; } } } - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) + MilliSleep(500); + if (interruptNet.load() == true) { return; } @@ -1927,17 +1964,18 @@ void CConnman::ThreadOpenConnections() // Minimum time before next feeler connection (in microseconds). int64_t nNextFeeler = PoissonNextSend(nStart * 1000 * 1000, FEELER_INTERVAL); - while (!interruptNet) + while (interruptNet.load() == false) { ProcessOneShot(); - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) + MilliSleep(500); + if (interruptNet.load() == true) { return; } CSemaphoreGrant grant(*semOutbound); - if (interruptNet) + if (interruptNet.load() == true) { return; } @@ -2015,7 +2053,7 @@ void CConnman::ThreadOpenConnections() int64_t nANow = GetAdjustedTime(); int nTries = 0; - while (!interruptNet) + while (interruptNet.load() == false) { CAddrInfo addr = addrman.Select(fFeeler); @@ -2079,9 +2117,12 @@ void CConnman::ThreadOpenConnections() // Add small amount of random noise before connection to avoid // synchronization. int randsleep = GetRandInt(FEELER_SLEEP_WINDOW * 1000); - if (!interruptNet.sleep_for(std::chrono::milliseconds(randsleep))) + MilliSleep(randsleep); { - return; + if (interruptNet.load() == true) + { + return; + } } LogPrintf("Making feeler connection to %s\n", addrConnect.ToString()); } @@ -2196,7 +2237,8 @@ void CConnman::ThreadOpenAddedConnections() LookupNumeric(info.strAddedNode.c_str(), pnetMan->getActivePaymentNetwork()->GetDefaultPort())); OpenNetworkConnection( CAddress(service, NODE_NONE), false, &grant, info.strAddedNode.c_str(), false, false, true); - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) + MilliSleep(500); + if (interruptNet.load() == true) { return; } @@ -2205,7 +2247,8 @@ void CConnman::ThreadOpenAddedConnections() } // Retry every 60 seconds if a connection was attempted, otherwise two // seconds. - if (!interruptNet.sleep_for(std::chrono::seconds(tried ? 60 : 2))) + MilliSleep((tried ? 60 : 2) * 1000); + if (interruptNet.load() == true) { return; } @@ -2224,7 +2267,7 @@ bool CConnman::OpenNetworkConnection(const CAddress &addrConnect, // // Initiate outbound network connection // - if (interruptNet) + if (interruptNet.load() == true) { return false; } @@ -2454,7 +2497,7 @@ bool CConnman::BindListenPort(const CService &addrBind, std::string &strError, b return true; } -void Discover(boost::thread_group &threadGroup) +void Discover(thread_group &threadGroup) { if (!fDiscover) { @@ -2551,10 +2594,11 @@ CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : nSeed0(nSeed0In), nSe nBestHeight = 0; clientInterface = nullptr; flagInterruptMsgProc = false; + interruptNet.store(false); } NodeId CConnman::GetNewNodeId() { return nLastNodeId.fetch_add(1, std::memory_order_relaxed); } -bool CConnman::Start(CScheduler &scheduler, std::string &strNodeError, Options connOptions) +bool CConnman::Start(std::string &strNodeError, Options connOptions) { nTotalBytesRecv = 0; nTotalBytesSent = 0; @@ -2645,8 +2689,8 @@ bool CConnman::Start(CScheduler &scheduler, std::string &strNodeError, Options c // Start threads // InterruptSocks5(false); - interruptNet.reset(); flagInterruptMsgProc = false; + interruptNet.store(false); { std::unique_lock lock(mutexMsgProc); @@ -2683,7 +2727,8 @@ bool CConnman::Start(CScheduler &scheduler, std::string &strNodeError, Options c std::function(std::bind(&CConnman::ThreadMessageHandler, this))); // Dump network addresses - scheduler.scheduleEvery(std::bind(&CConnman::DumpData, this), DUMP_ADDRESSES_INTERVAL * 1000); + threadDumpData = std::thread(&TraceThread >, "dumpdata", + std::function(std::bind(&CConnman::DumpData, this, DUMP_ADDRESSES_INTERVAL))); return true; } @@ -2708,8 +2753,7 @@ void CConnman::Interrupt() flagInterruptMsgProc = true; } condMsgProc.notify_all(); - - interruptNet(); + interruptNet.store(true); InterruptSocks5(true); if (semOutbound) @@ -2751,10 +2795,14 @@ void CConnman::Stop() { threadSocketHandler.join(); } + if (threadDumpData.joinable()) + { + threadDumpData.join(); + } if (fAddressesInitialized) { - DumpData(); + _DumpData(); fAddressesInitialized = false; } diff --git a/src/net.h b/src/net.h index 30290a10..2f98fcea 100644 --- a/src/net.h +++ b/src/net.h @@ -41,7 +41,7 @@ #include "random.h" #include "streams.h" #include "sync.h" -#include "threadinterrupt.h" +#include "threadgroup.h" #include "uint256.h" #ifndef WIN32 @@ -172,7 +172,7 @@ class CConnman }; CConnman(uint64_t seed0, uint64_t seed1); ~CConnman(); - bool Start(CScheduler &scheduler, std::string &strNodeError, Options options); + bool Start(std::string &strNodeError, Options options); void Stop(); void Interrupt(); bool BindListenPort(const CService &bindAddr, std::string &strError, bool fWhitelisted = false); @@ -358,7 +358,8 @@ class CConnman //! clean unused entries (if bantime has expired) void SweepBanned(); void DumpAddresses(); - void DumpData(); + void _DumpData(); + void DumpData(int64_t seconds_between_runs); void DumpBanlist(); // Network stats @@ -429,17 +430,18 @@ class CConnman std::mutex mutexMsgProc; std::atomic flagInterruptMsgProc; - CThreadInterrupt interruptNet; + std::atomic interruptNet; std::thread threadDNSAddressSeed; std::thread threadSocketHandler; std::thread threadOpenAddedConnections; std::thread threadOpenConnections; std::thread threadMessageHandler; + std::thread threadDumpData; }; extern std::unique_ptr g_connman; -void Discover(boost::thread_group &threadGroup); +void Discover(thread_group &threadGroup); void MapPort(bool fUseUPnP); unsigned short GetListenPort(); bool BindListenPort(const CService &bindAddr, std::string &strError, bool fWhitelisted = false); diff --git a/src/netbase.cpp b/src/netbase.cpp index d3c53dde..f7d5e75a 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -46,7 +46,7 @@ #include // for to_lower() #include // for startswith() and endswith() -#include + #if !defined(HAVE_MSG_NOSIGNAL) && !defined(MSG_NOSIGNAL) #define MSG_NOSIGNAL 0 diff --git a/src/processblock.cpp b/src/processblock.cpp index 3c666438..3e5f9d37 100644 --- a/src/processblock.cpp +++ b/src/processblock.cpp @@ -24,7 +24,7 @@ #include #include #include -#include + #include #include "args.h" @@ -678,7 +678,11 @@ bool ActivateBestChain(CValidationState &state, CBlockIndex *pindexMostWork = NULL; do { - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + break; + } + if (ShutdownRequested()) break; @@ -1212,6 +1216,8 @@ void ThreadScriptCheck() RenameThread("bitcoin-scriptch"); scriptcheckqueue.Thread(); } + +void InterruptScriptCheck() { scriptcheckqueue.Stop(); } static int64_t nTimeCheck = 0; static int64_t nTimeForks = 0; static int64_t nTimeVerify = 0; @@ -1456,12 +1462,9 @@ bool ConnectBlock(const CBlock &block, return state.DoS(100, error("ConnectBlock(): coinstake pays too much"), REJECT_INVALID, "bad-cb-amount"); } } -retry: if (!control.Wait()) { - MilliSleep(50); - goto retry; - // return state.DoS(100, false); + return state.DoS(100, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); } int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; diff --git a/src/processblock.h b/src/processblock.h index 39fc3066..e904b852 100644 --- a/src/processblock.h +++ b/src/processblock.h @@ -50,6 +50,7 @@ bool UndoReadFromDisk(CBlockUndo &blockundo, const CDiskBlockPos &pos, const uin /** Run an instance of the script checking thread */ void ThreadScriptCheck(); +void InterruptScriptCheck(); /** Apply the effects of this block (with given index) on the UTXO set represented by coins */ bool ConnectBlock(const CBlock &block, diff --git a/src/rpc/rpcblockchain.cpp b/src/rpc/rpcblockchain.cpp index 20d578fd..d1b11fb0 100644 --- a/src/rpc/rpcblockchain.cpp +++ b/src/rpc/rpcblockchain.cpp @@ -43,8 +43,6 @@ #include -#include // boost::thread::interrupt - extern void TxToJSON(const CTransaction &tx, const uint256 hashBlock, UniValue &entry); void ScriptPubKeyToJSON(const CScript &scriptPubKey, UniValue &out, bool fIncludeHex); @@ -489,7 +487,11 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) std::map outputs; while (pcursor->Valid()) { - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + LogPrintf("GetUTXOStats(): Shutdown requested. Exiting.\n"); + return false; + } COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) diff --git a/src/scheduler.cpp b/src/scheduler.cpp deleted file mode 100644 index abab4764..00000000 --- a/src/scheduler.cpp +++ /dev/null @@ -1,146 +0,0 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2015-2016 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include "scheduler.h" - -#include "reverselock.h" - -#include -#include -#include - -CScheduler::CScheduler() : nThreadsServicingQueue(0), stopRequested(false), stopWhenEmpty(false) {} -CScheduler::~CScheduler() { assert(nThreadsServicingQueue == 0); } -#if BOOST_VERSION < 105000 -static boost::system_time toPosixTime(const boost::chrono::system_clock::time_point &t) -{ - return boost::posix_time::from_time_t(boost::chrono::system_clock::to_time_t(t)); -} -#endif - -void CScheduler::serviceQueue() -{ - boost::unique_lock lock(newTaskMutex); - ++nThreadsServicingQueue; - - // newTaskMutex is locked throughout this loop EXCEPT - // when the thread is waiting or when the user's function - // is called. - while (!shouldStop()) - { - try - { - while (!shouldStop() && taskQueue.empty()) - { - // Wait until there is something to do. - newTaskScheduled.wait(lock); - } - -// Wait until either there is a new task, or until -// the time of the first item on the queue: - -// wait_until needs boost 1.50 or later; older versions have timed_wait: -#if BOOST_VERSION < 105000 - while (!shouldStop() && !taskQueue.empty() && - newTaskScheduled.timed_wait(lock, toPosixTime(taskQueue.begin()->first))) - { - // Keep waiting until timeout - } -#else - // Some boost versions have a conflicting overload of wait_until that returns void. - // Explicitly use a template here to avoid hitting that overload. - while (!shouldStop() && !taskQueue.empty() && - newTaskScheduled.wait_until<>(lock, taskQueue.begin()->first) != boost::cv_status::timeout) - { - // Keep waiting until timeout - } -#endif - // If there are multiple threads, the queue can empty while we're waiting (another - // thread may service the task we were waiting on). - if (shouldStop() || taskQueue.empty()) - continue; - - Function f = taskQueue.begin()->second; - taskQueue.erase(taskQueue.begin()); - - { - // Unlock before calling f, so it can reschedule itself or another task - // without deadlocking: - reverse_lock > rlock(lock); - f(); - } - } - catch (...) - { - --nThreadsServicingQueue; - throw; - } - } - --nThreadsServicingQueue; -} - -void CScheduler::stop(bool drain) -{ - { - boost::unique_lock lock(newTaskMutex); - if (drain) - stopWhenEmpty = true; - else - stopRequested = true; - } - newTaskScheduled.notify_all(); -} - -void CScheduler::schedule(CScheduler::Function f, boost::chrono::system_clock::time_point t) -{ - { - boost::unique_lock lock(newTaskMutex); - taskQueue.insert(std::make_pair(t, f)); - } - newTaskScheduled.notify_one(); -} - -void CScheduler::scheduleFromNow(CScheduler::Function f, int64_t deltaSeconds) -{ - schedule(f, boost::chrono::system_clock::now() + boost::chrono::seconds(deltaSeconds)); -} - -static void Repeat(CScheduler *s, CScheduler::Function f, int64_t deltaSeconds) -{ - f(); - s->scheduleFromNow(boost::bind(&Repeat, s, f, deltaSeconds), deltaSeconds); -} - -void CScheduler::scheduleEvery(CScheduler::Function f, int64_t deltaSeconds) -{ - scheduleFromNow(boost::bind(&Repeat, this, f, deltaSeconds), deltaSeconds); -} - -size_t CScheduler::getQueueInfo(boost::chrono::system_clock::time_point &first, - boost::chrono::system_clock::time_point &last) const -{ - boost::unique_lock lock(newTaskMutex); - size_t result = taskQueue.size(); - if (!taskQueue.empty()) - { - first = taskQueue.begin()->first; - last = taskQueue.rbegin()->first; - } - return result; -} diff --git a/src/scheduler.h b/src/scheduler.h deleted file mode 100644 index c7f91511..00000000 --- a/src/scheduler.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2015-2016 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef BITCOIN_SCHEDULER_H -#define BITCOIN_SCHEDULER_H - -// -// NOTE: -// boost::thread / boost::function / boost::chrono should be ported to -// std::thread / std::function / std::chrono when we support C++11. -// -#include -#include -#include -#include - -// -// Simple class for background tasks that should be run -// periodically or once "after a while" -// -// Usage: -// -// CScheduler* s = new CScheduler(); -// s->scheduleFromNow(doSomething, 11); // Assuming a: void doSomething() { } -// s->scheduleFromNow(boost::bind(Class::func, this, argument), 3); -// boost::thread* t = new boost::thread(boost::bind(CScheduler::serviceQueue, s)); -// -// ... then at program shutdown, clean up the thread running serviceQueue: -// t->interrupt(); -// t->join(); -// delete t; -// delete s; // Must be done after thread is interrupted/joined. -// - -class CScheduler -{ -public: - CScheduler(); - ~CScheduler(); - - typedef boost::function Function; - - // Call func at/after time t - void schedule(Function f, boost::chrono::system_clock::time_point t); - - // Convenience method: call f once deltaSeconds from now - void scheduleFromNow(Function f, int64_t deltaSeconds); - - // Another convenience method: call f approximately - // every deltaSeconds forever, starting deltaSeconds from now. - // To be more precise: every time f is finished, it - // is rescheduled to run deltaSeconds later. If you - // need more accurate scheduling, don't use this method. - void scheduleEvery(Function f, int64_t deltaSeconds); - - // To keep things as simple as possible, there is no unschedule. - - // Services the queue 'forever'. Should be run in a thread, - // and interrupted using boost::interrupt_thread - void serviceQueue(); - - // Tell any threads running serviceQueue to stop as soon as they're - // done servicing whatever task they're currently servicing (drain=false) - // or when there is no work left to be done (drain=true) - void stop(bool drain = false); - - // Returns number of tasks waiting to be serviced, - // and first and last task times - size_t getQueueInfo(boost::chrono::system_clock::time_point &first, - boost::chrono::system_clock::time_point &last) const; - -private: - std::multimap taskQueue; - boost::condition_variable newTaskScheduled; - mutable boost::mutex newTaskMutex; - int nThreadsServicingQueue; - bool stopRequested; - bool stopWhenEmpty; - bool shouldStop() { return stopRequested || (stopWhenEmpty && taskQueue.empty()); } -}; - -#endif diff --git a/src/sync.cpp b/src/sync.cpp index 9789a82b..681ceaaf 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -23,9 +23,11 @@ #include "util/util.h" #include "util/utilstrencodings.h" +#include +#include +#include #include -#include #ifdef DEBUG_LOCKCONTENTION void PrintLockContention(const char *pszName, const char *pszFile, int nLine) @@ -72,11 +74,24 @@ struct CLockLocation }; typedef std::vector > LockStack; +typedef std::map, LockStack> LockOrders; +typedef std::set > InvLockOrders; -static boost::mutex dd_mutex; -static std::map, LockStack> lockorders; -static boost::thread_specific_ptr lockstack; - +struct LockData +{ + // Very ugly hack: as the global constructs and destructors run single + // threaded, we use this boolean to know whether LockData still exists, + // as DeleteLock can get called by global CCriticalSection destructors + // after LockData disappears. + bool available; + LockData() : available(true) {} + ~LockData() { available = false; } + LockOrders lockorders; + InvLockOrders invlockorders; + std::mutex dd_mutex; +} static lockdata; + +static thread_local LockStack g_lockstack; static void potential_deadlock_detected(const std::pair &mismatch, const LockStack &s1, @@ -136,59 +151,53 @@ static void potential_deadlock_detected(const std::pair &mismatc assert(onlyMaybeDeadlock); } -static void push_lock(void *c, const CLockLocation &locklocation, bool fTry) +static void push_lock(void *c, const CLockLocation &locklocation) { - if (lockstack.get() == NULL) - lockstack.reset(new LockStack); + std::lock_guard lock(lockdata.dd_mutex); - dd_mutex.lock(); + g_lockstack.push_back(std::make_pair(c, locklocation)); - (*lockstack).push_back(std::make_pair(c, locklocation)); - - if (!fTry) + for (const std::pair &i : g_lockstack) { - for (auto const &i : (*lockstack)) + if (i.first == c) { - if (i.first == c) - break; + break; + } - std::pair p1 = std::make_pair(i.first, c); - if (lockorders.count(p1)) - continue; - lockorders[p1] = (*lockstack); + std::pair p1 = std::make_pair(i.first, c); + if (lockdata.lockorders.count(p1)) + { + continue; + } + lockdata.lockorders[p1] = g_lockstack; - std::pair p2 = std::make_pair(c, i.first); - if (lockorders.count(p2)) - potential_deadlock_detected(p1, lockorders[p2], lockorders[p1]); + std::pair p2 = std::make_pair(c, i.first); + lockdata.invlockorders.insert(p2); + if (lockdata.lockorders.count(p2)) + { + potential_deadlock_detected(p1, lockdata.lockorders[p2], lockdata.lockorders[p1]); } } - dd_mutex.unlock(); -} - -static void pop_lock() -{ - dd_mutex.lock(); - (*lockstack).pop_back(); - dd_mutex.unlock(); } +static void pop_lock() { g_lockstack.pop_back(); } void EnterCritical(const char *pszName, const char *pszFile, int nLine, void *cs, bool fTry) { - push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry), fTry); + push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry)); } void LeaveCritical() { pop_lock(); } std::string LocksHeld() { std::string result; - for (auto const &i : *lockstack) + for (const std::pair &i : g_lockstack) result += i.second.ToString() + std::string("\n"); return result; } void AssertLockHeldInternal(const char *pszName, const char *pszFile, int nLine, void *cs) { - for (auto const &i : *lockstack) + for (const std::pair &i : g_lockstack) if (i.first == cs) return; fprintf(stderr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, @@ -196,4 +205,44 @@ void AssertLockHeldInternal(const char *pszName, const char *pszFile, int nLine, abort(); } +void AssertLockNotHeldInternal(const char *pszName, const char *pszFile, int nLine, void *cs) +{ + for (const std::pair &i : g_lockstack) + { + if (i.first == cs) + { + fprintf(stderr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, + LocksHeld().c_str()); + abort(); + } + } +} + +void DeleteLock(void *cs) +{ + if (!lockdata.available) + { + // We're already shutting down. + return; + } + std::lock_guard lock(lockdata.dd_mutex); + std::pair item = std::make_pair(cs, nullptr); + LockOrders::iterator it = lockdata.lockorders.lower_bound(item); + while (it != lockdata.lockorders.end() && it->first.first == cs) + { + std::pair invitem = std::make_pair(it->first.second, it->first.first); + lockdata.invlockorders.erase(invitem); + lockdata.lockorders.erase(it++); + } + InvLockOrders::iterator invit = lockdata.invlockorders.lower_bound(item); + while (invit != lockdata.invlockorders.end() && invit->first == cs) + { + std::pair invinvitem = std::make_pair(invit->second, invit->first); + lockdata.lockorders.erase(invinvitem); + lockdata.invlockorders.erase(invit++); + } +} + +bool g_debug_lockorder_abort = true; + #endif /* DEBUG_LOCKORDER */ diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp deleted file mode 100644 index 97d366d5..00000000 --- a/src/test/scheduler_tests.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers -// Copyright (c) 2015-2017 The Bitcoin Unlimited developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "scheduler.h" -#include "random.h" - -#include "test/test_bitcoin.h" - -#include -#include -#include -#include -#include - -BOOST_AUTO_TEST_SUITE(scheduler_tests) - -static void microTask(CScheduler &s, - boost::mutex &mutex, - int &counter, - int delta, - boost::chrono::system_clock::time_point rescheduleTime) -{ - { - boost::unique_lock lock(mutex); - counter += delta; - } - boost::chrono::system_clock::time_point noTime = boost::chrono::system_clock::time_point::min(); - if (rescheduleTime != noTime) - { - CScheduler::Function f = - boost::bind(µTask, boost::ref(s), boost::ref(mutex), boost::ref(counter), -delta + 1, noTime); - s.schedule(f, rescheduleTime); - } -} - -static void MicroSleep(uint64_t n) -{ -#if defined(HAVE_WORKING_BOOST_SLEEP_FOR) - boost::this_thread::sleep_for(boost::chrono::microseconds(n)); -#elif defined(HAVE_WORKING_BOOST_SLEEP) - boost::this_thread::sleep(boost::posix_time::microseconds(n)); -#else -// should never get here -#error missing boost sleep implementation -#endif -} - -BOOST_AUTO_TEST_CASE(manythreads) -{ - seed_insecure_rand(false); - - // Stress test: hundreds of microsecond-scheduled tasks, - // serviced by 10 threads. - // - // So... ten shared counters, which if all the tasks execute - // properly will sum to the number of tasks done. - // Each task adds or subtracts from one of the counters a - // random amount, and then schedules another task 0-1000 - // microseconds in the future to subtract or add from - // the counter -random_amount+1, so in the end the shared - // counters should sum to the number of initial tasks performed. - CScheduler microTasks; - - boost::mutex counterMutex[10]; - int counter[10] = {0}; - boost::random::mt19937 rng(insecure_rand()); - boost::random::uniform_int_distribution<> zeroToNine(0, 9); - boost::random::uniform_int_distribution<> randomMsec(-11, 1000); - boost::random::uniform_int_distribution<> randomDelta(-1000, 1000); - - boost::chrono::system_clock::time_point start = boost::chrono::system_clock::now(); - boost::chrono::system_clock::time_point now = start; - boost::chrono::system_clock::time_point first, last; - size_t nTasks = microTasks.getQueueInfo(first, last); - BOOST_CHECK(nTasks == 0); - - for (int i = 0; i < 100; i++) - { - boost::chrono::system_clock::time_point t = now + boost::chrono::microseconds(randomMsec(rng)); - boost::chrono::system_clock::time_point tReschedule = now + boost::chrono::microseconds(500 + randomMsec(rng)); - int whichCounter = zeroToNine(rng); - CScheduler::Function f = boost::bind(µTask, boost::ref(microTasks), boost::ref(counterMutex[whichCounter]), - boost::ref(counter[whichCounter]), randomDelta(rng), tReschedule); - microTasks.schedule(f, t); - } - nTasks = microTasks.getQueueInfo(first, last); - BOOST_CHECK(nTasks == 100); - BOOST_CHECK(first < last); - BOOST_CHECK(last > now); - - // As soon as these are created they will start running and servicing the queue - boost::thread_group microThreads; - for (int i = 0; i < 5; i++) - microThreads.create_thread(boost::bind(&CScheduler::serviceQueue, µTasks)); - - MicroSleep(600); - now = boost::chrono::system_clock::now(); - - // More threads and more tasks: - for (int i = 0; i < 5; i++) - microThreads.create_thread(boost::bind(&CScheduler::serviceQueue, µTasks)); - for (int i = 0; i < 100; i++) - { - boost::chrono::system_clock::time_point t = now + boost::chrono::microseconds(randomMsec(rng)); - boost::chrono::system_clock::time_point tReschedule = now + boost::chrono::microseconds(500 + randomMsec(rng)); - int whichCounter = zeroToNine(rng); - CScheduler::Function f = boost::bind(µTask, boost::ref(microTasks), boost::ref(counterMutex[whichCounter]), - boost::ref(counter[whichCounter]), randomDelta(rng), tReschedule); - microTasks.schedule(f, t); - } - - // Drain the task queue then exit threads - microTasks.stop(true); - microThreads.join_all(); // ... wait until all the threads are done - - int counterSum = 0; - for (int i = 0; i < 10; i++) - { - BOOST_CHECK(counter[i] != 0); - counterSum += counter[i]; - } - BOOST_CHECK_EQUAL(counterSum, 200); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 81694d00..91c32386 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -30,7 +30,6 @@ #include -#include extern bool fPrintToConsole; extern void noui_connect(); diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index bc23ea04..b0bdb259 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -10,6 +10,7 @@ #include + extern CNetworkManager *pnetman; /** Basic testing setup. diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 94c6a7c1..6a7ce1e9 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE(util_sharedcriticalsection) } while (0); { // If the read lock does not allow simultaneous locking, this code will hang in the join_all - boost::thread_group thrds; + thread_group thrds; READLOCK(cs); thrds.create_thread(boost::bind(ThreadSharedCritTest, &cs)); thrds.join_all(); @@ -90,7 +90,7 @@ BOOST_AUTO_TEST_CASE(util_sharedcriticalsection) threadExited = false; readVal = 0; critVal = 1; - boost::thread_group thrds; + thread_group thrds; { WRITELOCK(cs); thrds.create_thread(boost::bind(ThreadSharedCritTest, &cs)); @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(util_threadcorral) CThreadCorral corral; { // ensure that regions lock out other regions, but not the current region. - boost::thread_group thrds; + thread_group thrds; int readVals[3] = {0, 0, 0}; { CORRAL(corral, 1); diff --git a/src/threadgroup.h b/src/threadgroup.h new file mode 100644 index 00000000..a1061890 --- /dev/null +++ b/src/threadgroup.h @@ -0,0 +1,42 @@ +#ifndef ECCOIN_THREAD_GROUP_H +#define ECCOIN_THREAD_GROUP_H + +#include +#include +#include + + +extern std::atomic shutdown_threads; + +class thread_group +{ +private: + std::vector threads; + std::vector names; + +public: + void interrupt_all() { shutdown_threads.store(true); } + template + void create_thread(std::string name, Fn &&f, Args &&... args) + { + threads.push_back(std::thread(f, args...)); + names.push_back(name); + } + + bool empty() { return threads.empty(); } + void join_all() + { + // printf("num threads = %u \n", threads.size()); + for (size_t i = 0; i < threads.size(); i++) + { + // printf("trying to join thread %s \n", names[i].c_str()); + if (threads[i].joinable()) + { + threads[i].join(); + } + // printf("joined thread %s \n", names[i].c_str()); + } + } +}; + +#endif diff --git a/src/threadinterrupt.cpp b/src/threadinterrupt.cpp deleted file mode 100644 index 7d22a9d0..00000000 --- a/src/threadinterrupt.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2009-2010 Satoshi Nakamoto - * Copyright (c) 2009-2016 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include "threadinterrupt.h" - -CThreadInterrupt::operator bool() const { return flag.load(std::memory_order_acquire); } -void CThreadInterrupt::reset() { flag.store(false, std::memory_order_release); } -void CThreadInterrupt::operator()() -{ - { - std::unique_lock lock(mut); - flag.store(true, std::memory_order_release); - } - cond.notify_all(); -} - -bool CThreadInterrupt::sleep_for(std::chrono::milliseconds rel_time) -{ - std::unique_lock lock(mut); - return !cond.wait_for(lock, rel_time, [this]() { return flag.load(std::memory_order_acquire); }); -} - -bool CThreadInterrupt::sleep_for(std::chrono::seconds rel_time) -{ - return sleep_for(std::chrono::duration_cast(rel_time)); -} - -bool CThreadInterrupt::sleep_for(std::chrono::minutes rel_time) -{ - return sleep_for(std::chrono::duration_cast(rel_time)); -} diff --git a/src/threadinterrupt.h b/src/threadinterrupt.h deleted file mode 100644 index 815bc015..00000000 --- a/src/threadinterrupt.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2009-2010 Satoshi Nakamoto - * Copyright (c) 2009-2016 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef BITCOIN_THREADINTERRUPT_H -#define BITCOIN_THREADINTERRUPT_H - -#include -#include -#include -#include - -/** - * A helper class for interruptible sleeps. Calling operator() will interrupt - * any current sleep, and after that point operator bool() will return true - * until reset. - */ -class CThreadInterrupt -{ -public: - explicit operator bool() const; - void operator()(); - void reset(); - bool sleep_for(std::chrono::milliseconds rel_time); - bool sleep_for(std::chrono::seconds rel_time); - bool sleep_for(std::chrono::minutes rel_time); - -private: - std::condition_variable cond; - std::mutex mut; - std::atomic flag; -}; - -#endif // BITCOIN_THREADINTERRUPT_H diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 390a8085..91ec0d57 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -753,7 +753,7 @@ void TorController::reconnect_cb(evutil_socket_t fd, short what, void *arg) /****** Thread ********/ struct event_base *base; -boost::thread torControlThread; +std::thread torControlThread; static void TorControlThread() { @@ -762,7 +762,7 @@ static void TorControlThread() event_base_dispatch(base); } -void StartTorControl(boost::thread_group &threadGroup, CScheduler &scheduler) +void StartTorControl(thread_group &threadGroup) { assert(!base); #ifdef WIN32 @@ -777,7 +777,7 @@ void StartTorControl(boost::thread_group &threadGroup, CScheduler &scheduler) return; } - torControlThread = boost::thread(boost::bind(&TraceThread, "torcontrol", &TorControlThread)); + torControlThread = std::thread(boost::bind(&TraceThread, "torcontrol", &TorControlThread)); } void InterruptTorControl() diff --git a/src/torcontrol.h b/src/torcontrol.h index 00c97748..803b2609 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -23,12 +23,12 @@ #ifndef BITCOIN_TORCONTROL_H #define BITCOIN_TORCONTROL_H -#include "scheduler.h" +#include "threadgroup.h" extern const std::string DEFAULT_TOR_CONTROL; static const bool DEFAULT_LISTEN_ONION = true; -void StartTorControl(boost::thread_group &threadGroup, CScheduler &scheduler); +void StartTorControl(thread_group &threadGroup); void InterruptTorControl(); void StopTorControl(); diff --git a/src/txdb.cpp b/src/txdb.cpp index ae254dc3..56a123fc 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -32,7 +32,6 @@ #include -#include static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; @@ -226,7 +225,11 @@ bool CBlockTreeDB::LoadBlockIndexGuts() // Load mapBlockIndex while (pcursor->Valid()) { - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + LogPrintf("LoadBlockIndexGuts(): Shutdown requested. Exiting.\n"); + return false; + } std::pair key; if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { @@ -375,7 +378,11 @@ bool CCoinsViewDB::Upgrade() std::pair prev_key = {DB_COINS, uint256()}; while (pcursor->Valid()) { - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + LogPrintf("CCoinsViewDB::Upgrade(): Shutdown requested. Exiting.\n"); + return false; + } if (pcursor->GetKey(key) && key.first == DB_COINS) { diff --git a/src/util/util.cpp b/src/util/util.cpp index aee03cd0..007297e2 100644 --- a/src/util/util.cpp +++ b/src/util/util.cpp @@ -91,6 +91,7 @@ #include #include #include + #include #include #include @@ -740,15 +741,7 @@ void SetThreadPriority(int nPriority) #endif // WIN32 } -int GetNumCores() -{ -#if BOOST_VERSION >= 105600 - return boost::thread::physical_concurrency(); -#else // Must fall back to hardware_concurrency, which unfortunately counts virtual cores - return boost::thread::hardware_concurrency(); -#endif -} - +int GetNumCores() { return std::thread::hardware_concurrency(); } bool WildcardMatch(const char *psz, const char *mask) { while (true) diff --git a/src/util/utiltime.cpp b/src/util/utiltime.cpp index ac9a6476..f4707734 100644 --- a/src/util/utiltime.cpp +++ b/src/util/utiltime.cpp @@ -19,9 +19,11 @@ */ #include "util/utiltime.h" +#include +#include #include -#include + static int64_t nMockTime = 0; //! For unit testing @@ -64,7 +66,7 @@ int64_t GetLogTimeMicros() return GetTimeMicros(); } -void MilliSleep(int64_t n) { boost::this_thread::sleep_for(boost::chrono::milliseconds(n)); } +void MilliSleep(int64_t n) { std::this_thread::sleep_for(std::chrono::milliseconds(n)); } std::string DateTimeStrFormat(const char *pszFormat, int64_t nTime) { // std::locale takes ownership of the pointer diff --git a/src/verifydb.cpp b/src/verifydb.cpp index 8ef94dd5..098d8490 100644 --- a/src/verifydb.cpp +++ b/src/verifydb.cpp @@ -18,12 +18,11 @@ * along with this program. If not, see . */ -#include +#include "verifydb.h" #include "init.h" #include "main.h" #include "processblock.h" -#include "verifydb.h" CVerifyDB::CVerifyDB() { uiInterface.ShowProgress(_("Verifying blocks..."), 0); } @@ -50,7 +49,11 @@ bool CVerifyDB::VerifyDB(const CNetworkTemplate &chainparams, CCoinsView *coinsv for (CBlockIndex *pindex = pnetMan->getChainActive()->chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + LogPrintf("VerifyDB(): Shutdown requested. Exiting.\n"); + return false; + } uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min( 99, (int)(((double)(pnetMan->getChainActive()->chainActive.Height() - pindex->nHeight)) / @@ -112,7 +115,11 @@ bool CVerifyDB::VerifyDB(const CNetworkTemplate &chainparams, CCoinsView *coinsv CBlockIndex *pindex = pindexState; while (pindex != pnetMan->getChainActive()->chainActive.Tip()) { - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + LogPrintf("VerifyDB(): [lower] Shutdown requested. Exiting.\n"); + return false; + } uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, 100 - (int)(((double)(pnetMan->getChainActive()->chainActive.Height() - pindex->nHeight)) / diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 8e972fca..8af471b3 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -24,6 +24,7 @@ #include "args.h" #include "crypto/hash.h" #include "protocol.h" +#include "threadgroup.h" #include "util/util.h" #include "util/utilstrencodings.h" @@ -80,7 +81,9 @@ void CDBEnv::Close() { EnvShutdown(); } bool CDBEnv::Open(const fs::path &pathIn) { if (fDbEnvInit) + { return true; + } boost::this_thread::interruption_point(); @@ -118,7 +121,9 @@ bool CDBEnv::Open(const fs::path &pathIn) void CDBEnv::MakeMock() { if (fDbEnvInit) + { throw std::runtime_error("CDBEnv::MakeMock: Already initialized"); + } boost::this_thread::interruption_point(); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index efd61620..7ef0c5ac 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -46,7 +46,7 @@ #include #include -#include + const char *DEFAULT_WALLET_DAT = "wallet.dat"; @@ -775,7 +775,7 @@ bool CWallet::AddToWallet(const CWalletTx &wtxIn, bool fFromLoadWallet, CWalletD if (!strCmd.empty()) { boost::replace_all(strCmd, "%s", wtxIn.tx->GetHash().GetHex()); - boost::thread t(runCommand, strCmd); // thread runs free + std::thread t(runCommand, strCmd); // thread runs free } } return true; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index fbf557a7..ce64fdf2 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -34,7 +34,7 @@ #include #include -#include + static uint64_t nAccountingEntryNumber = 0; @@ -737,6 +737,10 @@ void ThreadFlushWalletDB(const std::string &strFile) while (true) { MilliSleep(500); + if (shutdown_threads.load()) + { + return; + } if (nLastSeen != nWalletDBUpdated) { @@ -760,7 +764,10 @@ void ThreadFlushWalletDB(const std::string &strFile) if (nRefCount == 0) { - boost::this_thread::interruption_point(); + if (shutdown_threads.load()) + { + return; + } std::map::iterator mi = bitdb.mapFileUseCount.find(strFile); if (mi != bitdb.mapFileUseCount.end()) { @@ -778,6 +785,10 @@ void ThreadFlushWalletDB(const std::string &strFile) } } } + if (shutdown_threads.load()) + { + return; + } } } From a1cc1c1863c08996cf882bdb3851b4bf14fcf209 Mon Sep 17 00:00:00 2001 From: Henry Young <8385576+HenryYoung42@users.noreply.github.com> Date: Wed, 16 Jan 2019 16:15:38 +0000 Subject: [PATCH 02/46] Changes to fix compiler warnings (#121) * Fixed log message error "stake" -> "work" in EccMiner * Fixed shadow variable nVersion in CBlockIndex serialization * Fixed shadow variable page_size in LockedPageManagerBase constructor * Fixed shadow variable fFlushOnClose in CWalletDB constructor * Fixed shadow variable vch in CPubKey constructor (using code from Bitcoin) * Fixed shadow variables - loops using i nested in a loop using i : inner loops -> j * Fixed shadow variable - n loop nested in n loop : inner loop -> m * Fixed shadow variable - constructor argument and member variable named the same - fixed per Bitcoin source * Fixed shadow variable - constructor argument and member variable named the same * Fixed shadow variables - fix taken from Bitcoin source * Fixed shadow variables - constructor arguments and member variables named the same * Fixed shadow variables and one uninitialized variable to tidy up compiler warnings * Fixed shadow variables and one unused variable * Fixed shadow variable pindexPrev * Fixed shadow variable - constructor argument and member variable named the same * Fixed shadow variables * Fixed shadow variables * Fixed shadow variables * Fixed shadow variables * Formatting corrections to comply with clang-format-3.8 --- src/blockgeneration/miner.cpp | 21 +++++++------- src/blockgeneration/minter.cpp | 12 ++++---- src/chain/blockindex.h | 6 ++-- src/httprpc.cpp | 2 +- src/httpserver.cpp | 16 +++++----- src/pubkey.cpp | 6 ++-- src/pubkey.h | 2 +- src/reverselock.h | 2 +- src/rpc/rpcmining.cpp | 4 +-- src/support/pagelocker.h | 2 +- src/test/coins_tests.cpp | 2 +- src/test/crypto_tests.cpp | 20 ++++++------- src/test/pmt_tests.cpp | 4 +-- src/torcontrol.cpp | 40 ++++++++++++------------- src/util/util.cpp | 12 ++++---- src/wallet/db.cpp | 6 ++-- src/wallet/test/wallet_tests.cpp | 6 ++-- src/wallet/wallet.cpp | 50 ++++++++++++++++---------------- src/wallet/walletdb.cpp | 3 -- src/wallet/walletdb.h | 4 +-- 20 files changed, 109 insertions(+), 111 deletions(-) diff --git a/src/blockgeneration/miner.cpp b/src/blockgeneration/miner.cpp index b5a3882e..0f59d094 100644 --- a/src/blockgeneration/miner.cpp +++ b/src/blockgeneration/miner.cpp @@ -151,7 +151,8 @@ std::unique_ptr CreateNewPoWBlock(CWallet *pwallet, const CScrip CTxMemPool::setEntries waitSet; // ppcoin: if coinstake available add coinstake tx - static int64_t nLastCoinStakeSearchTime = GetAdjustedTime(); // only initialized at startup + // Commented out unused variable assuming no side effect within GetAdjustedTime() + // static int64_t nLastCoinStakeSearchTime = GetAdjustedTime(); // only initialized at startup CBlockIndex *pindexPrev = pnetMan->getChainActive()->chainActive.Tip(); @@ -172,10 +173,10 @@ std::unique_ptr CreateNewPoWBlock(CWallet *pwallet, const CScrip // Collect memory pool transactions into the block { LOCK2(cs_main, mempool.cs); - CBlockIndex *pindexPrev = pnetMan->getChainActive()->chainActive.Tip(); - const int nHeight = pindexPrev->nHeight + 1; + CBlockIndex *_pindexPrev = pnetMan->getChainActive()->chainActive.Tip(); + const int nHeight = _pindexPrev->nHeight + 1; pblock->nTime = GetAdjustedTime(); - const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast(); + const int64_t nMedianTimePast = _pindexPrev->GetMedianTimePast(); pblock->nVersion = 4; @@ -323,12 +324,12 @@ std::unique_ptr CreateNewPoWBlock(CWallet *pwallet, const CScrip nLastBlockSize = nBlockSize; // Fill in header - pblock->hashPrevBlock = pindexPrev->GetBlockHash(); - pblock->nTime = std::max(pindexPrev->GetMedianTimePast() + 1, pblock->GetMaxTransactionTime()); - pblock->nTime = std::max(pblock->GetBlockTime(), pindexPrev->GetBlockTime() - nMaxClockDrift); - UpdateTime(pblock, pnetMan->getActivePaymentNetwork()->GetConsensus(), pindexPrev); + pblock->hashPrevBlock = _pindexPrev->GetBlockHash(); + pblock->nTime = std::max(_pindexPrev->GetMedianTimePast() + 1, pblock->GetMaxTransactionTime()); + pblock->nTime = std::max(pblock->GetBlockTime(), _pindexPrev->GetBlockTime() - nMaxClockDrift); + UpdateTime(pblock, pnetMan->getActivePaymentNetwork()->GetConsensus(), _pindexPrev); pblock->vtx[0]->vout[0].nValue = - GetProofOfWorkReward(nFees, pindexPrev->nHeight + 1, pindexPrev->GetBlockHash()); + GetProofOfWorkReward(nFees, _pindexPrev->nHeight + 1, _pindexPrev->GetBlockHash()); pblock->nNonce = 0; } @@ -422,7 +423,7 @@ bool CheckWork(const std::shared_ptr pblock, void EccMiner(CWallet *pwallet) { void *scratchbuf = scrypt_buffer_alloc(); - LogPrintf("CPUMiner started for proof-of-%s\n", "stake"); + LogPrintf("CPUMiner started for proof-of-%s\n", "work"); SetThreadPriority(THREAD_PRIORITY_LOWEST); // Make this thread recognisable as the mining thread RenameThread("ecc-miner"); diff --git a/src/blockgeneration/minter.cpp b/src/blockgeneration/minter.cpp index a0d0fc80..680125a4 100644 --- a/src/blockgeneration/minter.cpp +++ b/src/blockgeneration/minter.cpp @@ -166,10 +166,10 @@ std::unique_ptr CreateNewPoSBlock(CWallet *pwallet, const CScrip // Collect memory pool transactions into the block { LOCK2(cs_main, mempool.cs); - CBlockIndex *pindexPrev = pnetMan->getChainActive()->chainActive.Tip(); - const int nHeight = pindexPrev->nHeight + 1; + CBlockIndex *_pindexPrev = pnetMan->getChainActive()->chainActive.Tip(); + const int nHeight = _pindexPrev->nHeight + 1; pblock->nTime = GetAdjustedTime(); - const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast(); + const int64_t nMedianTimePast = _pindexPrev->GetMedianTimePast(); pblock->nVersion = 4; @@ -316,9 +316,9 @@ std::unique_ptr CreateNewPoSBlock(CWallet *pwallet, const CScrip nLastBlockSize = nBlockSize; // Fill in header - pblock->hashPrevBlock = pindexPrev->GetBlockHash(); - pblock->nTime = std::max(pindexPrev->GetMedianTimePast() + 1, pblock->GetMaxTransactionTime()); - pblock->nTime = std::max(pblock->GetBlockTime(), pindexPrev->GetBlockTime() - nMaxClockDrift); + pblock->hashPrevBlock = _pindexPrev->GetBlockHash(); + pblock->nTime = std::max(_pindexPrev->GetMedianTimePast() + 1, pblock->GetMaxTransactionTime()); + pblock->nTime = std::max(pblock->GetBlockTime(), _pindexPrev->GetBlockTime() - nMaxClockDrift); pblock->nNonce = 0; if (!pblock->IsProofOfStake()) { diff --git a/src/chain/blockindex.h b/src/chain/blockindex.h index 176e91f9..08f018a4 100644 --- a/src/chain/blockindex.h +++ b/src/chain/blockindex.h @@ -342,9 +342,9 @@ class CDiskBlockIndex : public CBlockIndex template inline void SerializationOp(Stream &s, Operation ser_action) { - int nVersion = s.GetVersion(); + int nStreamVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) - READWRITE(VARINT(nVersion, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(nStreamVersion, VarIntMode::NONNEGATIVE_SIGNED)); READWRITE(VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); READWRITE(VARINT(nStatus)); @@ -359,7 +359,7 @@ class CDiskBlockIndex : public CBlockIndex // added after db upgrade to reduce loadtime of index READWRITE(hashBlock); // block header - READWRITE(this->nVersion); + READWRITE(nVersion); READWRITE(hashPrev); READWRITE(hashMerkleRoot); READWRITE(nTime); diff --git a/src/httprpc.cpp b/src/httprpc.cpp index b0a2cee3..decf0004 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -62,7 +62,7 @@ class HTTPRPCTimer : public RPCTimerBase class HTTPRPCTimerInterface : public RPCTimerInterface { public: - HTTPRPCTimerInterface(struct event_base *base) : base(base) {} + HTTPRPCTimerInterface(struct event_base *_base) : base(_base) {} const char *Name() { return "HTTP"; } RPCTimerBase *NewTimer(boost::function &func, int64_t millis) { diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 99e18db9..29b4b4cd 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -62,8 +62,8 @@ static const size_t MAX_HEADERS_SIZE = 8192; class HTTPWorkItem : public HTTPClosure { public: - HTTPWorkItem(HTTPRequest *req, const std::string &path, const HTTPRequestHandler &func) - : req(req), path(path), func(func) + HTTPWorkItem(HTTPRequest *_req, const std::string &_path, const HTTPRequestHandler &_func) + : req(_req), path(_path), func(_func) { } void operator()() { func(req.get(), path); } @@ -109,7 +109,7 @@ class WorkQueue }; public: - WorkQueue(size_t maxDepth) : running(true), maxDepth(maxDepth), numThreads(0) {} + WorkQueue(size_t _maxDepth) : running(true), maxDepth(_maxDepth), numThreads(0) {} /*( Precondition: worker threads have all stopped * (call WaitExit) */ @@ -179,8 +179,8 @@ class WorkQueue struct HTTPPathHandler { HTTPPathHandler() {} - HTTPPathHandler(std::string prefix, bool exactMatch, HTTPRequestHandler handler) - : prefix(prefix), exactMatch(exactMatch), handler(handler) + HTTPPathHandler(std::string _prefix, bool _exactMatch, HTTPRequestHandler _handler) + : prefix(_prefix), exactMatch(_exactMatch), handler(_handler) { } std::string prefix; @@ -562,8 +562,8 @@ static void httpevent_callback_fn(evutil_socket_t, short, void *data) delete self; } -HTTPEvent::HTTPEvent(struct event_base *base, bool deleteWhenTriggered, const boost::function &handler) - : deleteWhenTriggered(deleteWhenTriggered), handler(handler) +HTTPEvent::HTTPEvent(struct event_base *base, bool _deleteWhenTriggered, const boost::function &_handler) + : deleteWhenTriggered(_deleteWhenTriggered), handler(_handler) { ev = event_new(base, -1, 0, httpevent_callback_fn, this); assert(ev); @@ -576,7 +576,7 @@ void HTTPEvent::trigger(struct timeval *tv) else evtimer_add(ev, tv); // trigger after timeval passed } -HTTPRequest::HTTPRequest(struct evhttp_request *req) : req(req), replySent(false) {} +HTTPRequest::HTTPRequest(struct evhttp_request *_req) : req(_req), replySent(false) {} HTTPRequest::~HTTPRequest() { if (!replySent) diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 48c6854f..84f5b109 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -359,13 +359,13 @@ void CExtPubKey::Decode(const unsigned char code[74]) pubkey.Set(code + 41, code + 74); } -bool CExtPubKey::Derive(CExtPubKey &out, unsigned int nChild) const +bool CExtPubKey::Derive(CExtPubKey &out, unsigned int _nChild) const { out.nDepth = nDepth + 1; CKeyID id = pubkey.GetID(); memcpy(&out.vchFingerprint[0], &id, 4); - out.nChild = nChild; - return pubkey.Derive(out.pubkey, out.vchChainCode, nChild, vchChainCode); + out.nChild = _nChild; + return pubkey.Derive(out.pubkey, out.vchChainCode, _nChild, vchChainCode); } diff --git a/src/pubkey.h b/src/pubkey.h index 1df76a9b..6cbce09f 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -93,7 +93,7 @@ class CPubKey } //! Construct a public key from a byte vector. - CPubKey(const std::vector &vch) { Set(vch.begin(), vch.end()); } + CPubKey(const std::vector &_vch) { Set(_vch.begin(), _vch.end()); } //! Simple read-only vector-like interface to the pubkey data. unsigned int size() const { return GetLen(vch[0]); } const unsigned char *begin() const { return vch; } diff --git a/src/reverselock.h b/src/reverselock.h index 2e6cc021..8902bd27 100644 --- a/src/reverselock.h +++ b/src/reverselock.h @@ -26,7 +26,7 @@ template class reverse_lock { public: - explicit reverse_lock(Lock &lock) : lock(lock) { lock.unlock(); } + explicit reverse_lock(Lock &_lock) : lock(_lock) { _lock.unlock(); } ~reverse_lock() { lock.lock(); } private: reverse_lock(reverse_lock const &); diff --git a/src/rpc/rpcmining.cpp b/src/rpc/rpcmining.cpp index 412713b5..13489cc6 100644 --- a/src/rpc/rpcmining.cpp +++ b/src/rpc/rpcmining.cpp @@ -428,8 +428,8 @@ UniValue setgenerate(const UniValue ¶ms, bool fHelp) "\nToggle the pow generation\n" + HelpExampleCli("setgenerate", "")); - if (pnetMan->getActivePaymentNetwork()->MineBlocksOnDemand()) - throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Use the generate method instead of setgenerate on this network"); + // if (pnetMan->getActivePaymentNetwork()->MineBlocksOnDemand()) + // throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Use the generate method instead of setgenerate on this network"); ThreadGeneration(pwalletMain, false, false); diff --git a/src/support/pagelocker.h b/src/support/pagelocker.h index 223ba66e..2baee8ac 100644 --- a/src/support/pagelocker.h +++ b/src/support/pagelocker.h @@ -43,7 +43,7 @@ template class LockedPageManagerBase { public: - LockedPageManagerBase(size_t page_size) : page_size(page_size) + LockedPageManagerBase(size_t p_size) : page_size(p_size) { // Determine bitmask for extracting page from address assert(!(page_size & (page_size - 1))); // size must be power of two diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index a113c58a..f22d788f 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -96,7 +96,7 @@ class CCoinsViewTest : public CCoinsView class CCoinsViewCacheTest : public CCoinsViewCache { public: - explicit CCoinsViewCacheTest(CCoinsView *base) : CCoinsViewCache(base) {} + explicit CCoinsViewCacheTest(CCoinsView *_base) : CCoinsViewCache(_base) {} void SelfTest() const { // Manually recompute the dynamic usage of the whole data, and compare it. diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 0b961551..6e89a4e5 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -143,13 +143,13 @@ void TestAES128CBC(const std::string &hexkey, { std::vector sub(i, in.end()); std::vector subout(sub.size() + AES_BLOCKSIZE); - int size = enc.Encrypt(&sub[0], sub.size(), &subout[0]); - if (size != 0) + int _size = enc.Encrypt(&sub[0], sub.size(), &subout[0]); + if (_size != 0) { - subout.resize(size); + subout.resize(_size); std::vector subdecrypted(subout.size()); - size = dec.Decrypt(&subout[0], subout.size(), &subdecrypted[0]); - subdecrypted.resize(size); + _size = dec.Decrypt(&subout[0], subout.size(), &subdecrypted[0]); + subdecrypted.resize(_size); BOOST_CHECK(decrypted.size() == in.size()); BOOST_CHECK_MESSAGE(subdecrypted == sub, HexStr(subdecrypted) + std::string(" != ") + HexStr(sub)); } @@ -188,13 +188,13 @@ void TestAES256CBC(const std::string &hexkey, { std::vector sub(i, in.end()); std::vector subout(sub.size() + AES_BLOCKSIZE); - int size = enc.Encrypt(&sub[0], sub.size(), &subout[0]); - if (size != 0) + int _size = enc.Encrypt(&sub[0], sub.size(), &subout[0]); + if (_size != 0) { - subout.resize(size); + subout.resize(_size); std::vector subdecrypted(subout.size()); - size = dec.Decrypt(&subout[0], subout.size(), &subdecrypted[0]); - subdecrypted.resize(size); + _size = dec.Decrypt(&subout[0], subout.size(), &subdecrypted[0]); + subdecrypted.resize(_size); BOOST_CHECK(decrypted.size() == in.size()); BOOST_CHECK_MESSAGE(subdecrypted == sub, HexStr(subdecrypted) + std::string(" != ") + HexStr(sub)); } diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index 4b73fd26..063c373c 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -86,8 +86,8 @@ BOOST_AUTO_TEST_CASE(pmt_test1) ss << pmt1; // verify CPartialMerkleTree's size guarantees - unsigned int n = std::min(nTx, 1 + vMatchTxid1.size() * nHeight); - BOOST_CHECK(ss.size() <= 10 + (258 * n + 7) / 8); + unsigned int m = std::min(nTx, 1 + vMatchTxid1.size() * nHeight); + BOOST_CHECK(ss.size() <= 10 + (258 * m + 7) / 8); // deserialize into a tester copy CPartialMerkleTreeTester pmt2; diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 91ec0d57..46b07594 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -137,7 +137,7 @@ class TorControlConnection static void eventcb(struct bufferevent *bev, short what, void *ctx); }; -TorControlConnection::TorControlConnection(struct event_base *base) : base(base), b_conn(0) {} +TorControlConnection::TorControlConnection(struct event_base *_base) : base(_base), b_conn(0) {} TorControlConnection::~TorControlConnection() { if (b_conn) @@ -217,8 +217,8 @@ void TorControlConnection::eventcb(struct bufferevent *bev, short what, void *ct } bool TorControlConnection::Connect(const std::string &target, - const ConnectionCB &connected, - const ConnectionCB &disconnected) + const ConnectionCB &_connected, + const ConnectionCB &_disconnected) { if (b_conn) Disconnect(); @@ -237,8 +237,8 @@ bool TorControlConnection::Connect(const std::string &target, return false; bufferevent_setcb(b_conn, TorControlConnection::readcb, NULL, TorControlConnection::eventcb, this); bufferevent_enable(b_conn, EV_READ | EV_WRITE); - this->connected = connected; - this->disconnected = disconnected; + this->connected = _connected; + this->disconnected = _disconnected; // Finally, connect to target if (bufferevent_socket_connect(b_conn, (struct sockaddr *)&connect_to_addr, connect_to_addrlen) < 0) @@ -431,12 +431,12 @@ class TorController static void reconnect_cb(evutil_socket_t fd, short what, void *arg); }; -TorController::TorController(struct event_base *base, const std::string &target) - : base(base), target(target), conn(base), reconnect(true), reconnect_ev(0), +TorController::TorController(struct event_base *_base, const std::string &_target) + : base(_base), target(_target), conn(base), reconnect(true), reconnect_ev(0), reconnect_timeout(RECONNECT_TIMEOUT_START) { // Start connection attempts immediately - if (!conn.Connect(target, boost::bind(&TorController::connected_cb, this, _1), + if (!conn.Connect(_target, boost::bind(&TorController::connected_cb, this, _1), boost::bind(&TorController::disconnected_cb, this, _1))) { LogPrintf("tor: Initiating connection to Tor control port %s failed\n", target); @@ -497,7 +497,7 @@ void TorController::add_onion_cb(TorControlConnection &_conn, const TorControlRe LogPrintf("tor: Add onion failed; error code %d\n", reply.code); } } -void TorController::auth_cb(TorControlConnection &conn, const TorControlReply &reply) +void TorController::auth_cb(TorControlConnection &_conn, const TorControlReply &reply) { if (reply.code == 250) { @@ -519,7 +519,7 @@ void TorController::auth_cb(TorControlConnection &conn, const TorControlReply &r // Request hidden service, redirect port. // Note that the 'virtual' port doesn't have to be the same as our internal port, but this is just a convenient // choice. TODO; refactor the shutdown sequence some day. - conn.Command(strprintf("ADD_ONION %s Port=%i,127.0.0.1:%i", private_key, GetListenPort(), GetListenPort()), + _conn.Command(strprintf("ADD_ONION %s Port=%i,127.0.0.1:%i", private_key, GetListenPort(), GetListenPort()), boost::bind(&TorController::add_onion_cb, this, _1, _2)); } else @@ -558,7 +558,7 @@ static std::vector ComputeResponse(const std::string &key, return computedHash; } -void TorController::authchallenge_cb(TorControlConnection &conn, const TorControlReply &reply) +void TorController::authchallenge_cb(TorControlConnection &_conn, const TorControlReply &reply) { if (reply.code == 250) { @@ -588,7 +588,7 @@ void TorController::authchallenge_cb(TorControlConnection &conn, const TorContro std::vector computedClientHash = ComputeResponse(TOR_SAFE_CLIENTKEY, cookie, clientNonce, serverNonce); - conn.Command( + _conn.Command( "AUTHENTICATE " + HexStr(computedClientHash), boost::bind(&TorController::auth_cb, this, _1, _2)); } else @@ -602,7 +602,7 @@ void TorController::authchallenge_cb(TorControlConnection &conn, const TorContro } } -void TorController::protocolinfo_cb(TorControlConnection &conn, const TorControlReply &reply) +void TorController::protocolinfo_cb(TorControlConnection &_conn, const TorControlReply &reply) { if (reply.code == 250) { @@ -650,7 +650,7 @@ void TorController::protocolinfo_cb(TorControlConnection &conn, const TorControl if (methods.count("NULL")) { LogPrint("tor", "tor: Using NULL authentication\n"); - conn.Command("AUTHENTICATE", boost::bind(&TorController::auth_cb, this, _1, _2)); + _conn.Command("AUTHENTICATE", boost::bind(&TorController::auth_cb, this, _1, _2)); } else if (methods.count("SAFECOOKIE")) { @@ -660,12 +660,12 @@ void TorController::protocolinfo_cb(TorControlConnection &conn, const TorControl std::pair status_cookie = ReadBinaryFile(cookiefile, TOR_COOKIE_SIZE); if (status_cookie.first && status_cookie.second.size() == TOR_COOKIE_SIZE) { - // conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), boost::bind(&TorController::auth_cb, + // _conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), boost::bind(&TorController::auth_cb, // this, _1, _2)); cookie = std::vector(status_cookie.second.begin(), status_cookie.second.end()); clientNonce = std::vector(TOR_NONCE_SIZE, 0); GetRandBytes(&clientNonce[0], TOR_NONCE_SIZE); - conn.Command("AUTHCHALLENGE SAFECOOKIE " + HexStr(clientNonce), + _conn.Command("AUTHCHALLENGE SAFECOOKIE " + HexStr(clientNonce), boost::bind(&TorController::authchallenge_cb, this, _1, _2)); } else @@ -687,7 +687,7 @@ void TorController::protocolinfo_cb(TorControlConnection &conn, const TorControl { LogPrint("tor", "tor: Using HASHEDPASSWORD authentication\n"); boost::replace_all(torpassword, "\"", "\\\""); - conn.Command( + _conn.Command( "AUTHENTICATE \"" + torpassword + "\"", boost::bind(&TorController::auth_cb, this, _1, _2)); } else @@ -706,15 +706,15 @@ void TorController::protocolinfo_cb(TorControlConnection &conn, const TorControl } } -void TorController::connected_cb(TorControlConnection &conn) +void TorController::connected_cb(TorControlConnection &_conn) { reconnect_timeout = RECONNECT_TIMEOUT_START; // First send a PROTOCOLINFO command to figure out what authentication is expected - if (!conn.Command("PROTOCOLINFO 1", boost::bind(&TorController::protocolinfo_cb, this, _1, _2))) + if (!_conn.Command("PROTOCOLINFO 1", boost::bind(&TorController::protocolinfo_cb, this, _1, _2))) LogPrintf("tor: Error sending initial protocolinfo command\n"); } -void TorController::disconnected_cb(TorControlConnection &conn) +void TorController::disconnected_cb(TorControlConnection &_conn) { // Stop advertizing service when disconnected if (service.IsValid()) diff --git a/src/util/util.cpp b/src/util/util.cpp index 007297e2..46b4aae1 100644 --- a/src/util/util.cpp +++ b/src/util/util.cpp @@ -505,19 +505,19 @@ bool TryCreateDirectory(const fs::path &p) return false; } -void FileCommit(FILE *fileout) +void FileCommit(FILE *_fileout) { - fflush(fileout); // harmless if redundantly called + fflush(_fileout); // harmless if redundantly called #ifdef WIN32 - HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(fileout)); + HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(_fileout)); FlushFileBuffers(hFile); #else #if defined(__linux__) || defined(__NetBSD__) - fdatasync(fileno(fileout)); + fdatasync(fileno(_fileout)); #elif defined(__APPLE__) && defined(F_FULLFSYNC) - fcntl(fileno(fileout), F_FULLFSYNC, 0); + fcntl(fileno(_fileout), F_FULLFSYNC, 0); #else - fsync(fileno(fileout)); + fsync(fileno(_fileout)); #endif #endif } diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 8af471b3..f7583c04 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -386,13 +386,13 @@ bool CDB::Rewrite(const std::string &strFile, const char *pszSkip) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT); - if (ret == DB_NOTFOUND) + int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT); + if (ret1 == DB_NOTFOUND) { pcursor->close(); break; } - else if (ret != 0) + else if (ret1 != 0) { pcursor->close(); fSuccess = false; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 94d4af81..d59b60d7 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) // run the 'mtgox' test (see http://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) // they tried to consolidate 10 50k coins into one 500k coin, and ended up with 50k in change empty_wallet(); - for (int i = 0; i < 20; i++) + for (int j = 0; j < 20; j++) add_coin(50000 * COIN); BOOST_CHECK( wallet.SelectCoinsMinConf(500000 * COIN, 1, 1, vCoins, setCoinsRet, nValueRet)); @@ -294,7 +294,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) BOOST_CHECK(!equal_sets(setCoinsRet, setCoinsRet2)); int fails = 0; - for (int i = 0; i < RANDOM_REPEATS; i++) + for (int j = 0; j < RANDOM_REPEATS; j++) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail @@ -311,7 +311,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) add_coin( 5*CENT); add_coin(10*CENT); add_coin(15*CENT); add_coin(20*CENT); add_coin(25*CENT); fails = 0; - for (int i = 0; i < RANDOM_REPEATS; i++) + for (int j = 0; j < RANDOM_REPEATS; j++) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 7ef0c5ac..318b331a 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -239,7 +239,7 @@ bool CWallet::LoadWatchOnly(const CScript &dest) { return CCryptoKeyStore::AddWa bool CWallet::Unlock(const SecureString &strWalletPassphrase) { CCrypter crypter; - CKeyingMaterial vMasterKey; + CKeyingMaterial _vMasterKey; { LOCK(cs_wallet); @@ -248,9 +248,9 @@ bool CWallet::Unlock(const SecureString &strWalletPassphrase) if (!crypter.SetKeyFromPassphrase(strWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod)) return false; - if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, vMasterKey)) + if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, _vMasterKey)) continue; // try another master key - if (CCryptoKeyStore::Unlock(vMasterKey)) + if (CCryptoKeyStore::Unlock(_vMasterKey)) return true; } } @@ -267,15 +267,15 @@ bool CWallet::ChangeWalletPassphrase(const SecureString &strOldWalletPassphrase, Lock(); CCrypter crypter; - CKeyingMaterial vMasterKey; + CKeyingMaterial _vMasterKey; for (auto &pMasterKey : mapMasterKeys) { if (!crypter.SetKeyFromPassphrase(strOldWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod)) return false; - if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, vMasterKey)) + if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, _vMasterKey)) return false; - if (CCryptoKeyStore::Unlock(vMasterKey)) + if (CCryptoKeyStore::Unlock(_vMasterKey)) { int64_t nStartTime = GetTimeMillis(); crypter.SetKeyFromPassphrase(strNewWalletPassphrase, pMasterKey.second.vchSalt, @@ -300,7 +300,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString &strOldWalletPassphrase, if (!crypter.SetKeyFromPassphrase(strNewWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod)) return false; - if (!crypter.Encrypt(vMasterKey, pMasterKey.second.vchCryptedKey)) + if (!crypter.Encrypt(_vMasterKey, pMasterKey.second.vchCryptedKey)) return false; CWalletDB(strWalletFile).WriteMasterKey(pMasterKey.first, pMasterKey.second); if (fWasLocked) @@ -375,8 +375,8 @@ std::set CWallet::GetConflicts(const uint256 &txid) const if (mapTxSpends.count(txin.prevout) <= 1) continue; // No conflict if zero or one spends range = mapTxSpends.equal_range(txin.prevout); - for (TxSpends::const_iterator it = range.first; it != range.second; ++it) - result.insert(it->second); + for (TxSpends::const_iterator _it = range.first; _it != range.second; ++_it) + result.insert(_it->second); } return result; } @@ -523,11 +523,11 @@ bool CWallet::EncryptWallet(const SecureString &strWalletPassphrase) if (IsCrypted()) return false; - CKeyingMaterial vMasterKey; + CKeyingMaterial _vMasterKey; RandAddSeedPerfmon(); - vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); - GetRandBytes(&vMasterKey[0], WALLET_CRYPTO_KEY_SIZE); + _vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); + GetRandBytes(&_vMasterKey[0], WALLET_CRYPTO_KEY_SIZE); CMasterKey kMasterKey; RandAddSeedPerfmon(); @@ -555,7 +555,7 @@ bool CWallet::EncryptWallet(const SecureString &strWalletPassphrase) if (!crypter.SetKeyFromPassphrase( strWalletPassphrase, kMasterKey.vchSalt, kMasterKey.nDeriveIterations, kMasterKey.nDerivationMethod)) return false; - if (!crypter.Encrypt(vMasterKey, kMasterKey.vchCryptedKey)) + if (!crypter.Encrypt(_vMasterKey, kMasterKey.vchCryptedKey)) return false; { @@ -574,7 +574,7 @@ bool CWallet::EncryptWallet(const SecureString &strWalletPassphrase) pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); } - if (!EncryptKeys(vMasterKey)) + if (!EncryptKeys(_vMasterKey)) { if (fFileBacked) { @@ -696,7 +696,7 @@ bool CWallet::AddToWallet(const CWalletTx &wtxIn, bool fFromLoadWallet, CWalletD CWalletTx *const pwtx = (*it).second; if (pwtx == &wtx) continue; - int64_t nSmartTime; + int64_t nSmartTime = 0; if (pwtx) { nSmartTime = pwtx->nTimeSmart; @@ -1167,9 +1167,9 @@ int CWalletTx::GetRequestCount() const // How about the block it's in? if (nRequests == 0 && !hashUnset()) { - std::map::const_iterator mi = pwallet->mapRequestCount.find(hashBlock); - if (mi != pwallet->mapRequestCount.end()) - nRequests = (*mi).second; + std::map::const_iterator mj = pwallet->mapRequestCount.find(hashBlock); + if (mj != pwallet->mapRequestCount.end()) + nRequests = (*mj).second; else nRequests = 1; // If it's in someone else's block it must have got out } @@ -1607,10 +1607,10 @@ bool CWalletTx::IsTrusted() const return true; } -bool CWalletTx::IsEquivalentTo(const CWalletTx &tx) const +bool CWalletTx::IsEquivalentTo(const CWalletTx &_tx) const { CTransaction tx1 = *(this->tx); - CTransaction tx2 = *(tx.tx); + CTransaction tx2 = *(_tx.tx); for (unsigned int i = 0; i < tx1.vin.size(); i++) tx1.vin[i].scriptSig = CScript(); for (unsigned int i = 0; i < tx2.vin.size(); i++) @@ -2908,17 +2908,17 @@ std::set > CWallet::GetAddressGroupings() std::set *> uniqueGroupings; // a set of pointers to groups of addresses std::map *> setmap; // map addresses to the unique group containing it - for (auto grouping : groupings) + for (auto _grouping : groupings) { // make a set of all the groups hit by this new group std::set *> hits; std::map *>::iterator it; - for (auto address : grouping) + for (auto address : _grouping) if ((it = setmap.find(address)) != setmap.end()) hits.insert((*it).second); // merge all hit groups into a new single group and delete old groups - std::set *merged = new std::set(grouping); + std::set *merged = new std::set(_grouping); for (auto *hit : hits) { merged->insert(hit->begin(), hit->end()); @@ -3611,14 +3611,14 @@ bool CWallet::CreateCoinStake(const CKeyStore &keystore, // Calculate coin age reward { uint64_t nCoinAge; - const CBlockIndex *pIndex0 = GetLastBlockIndex(pnetMan->getChainActive()->chainActive.Tip(), false); + const CBlockIndex *_pIndex0 = GetLastBlockIndex(pnetMan->getChainActive()->chainActive.Tip(), false); if (!txNew.GetCoinAge(nCoinAge)) { return error("CreateCoinStake : failed to calculate coin age"); } - int64_t nCreditReward = GetProofOfStakeReward(txNew.GetCoinAge(nCoinAge, true), pIndex0->nHeight); + int64_t nCreditReward = GetProofOfStakeReward(txNew.GetCoinAge(nCoinAge, true), _pIndex0->nHeight); LogPrintf("nCreditReward create=%d \n", nCreditReward); nCredit = nCredit + nCreditReward; } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index ce64fdf2..2e7b77d1 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -35,9 +35,6 @@ #include #include - -static uint64_t nAccountingEntryNumber = 0; - // // CWalletDB // diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index feab0a6a..19b6cbb7 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -89,8 +89,8 @@ class CKeyMetadata class CWalletDB : public CDB { public: - CWalletDB(const std::string &strFilename, const char *pszMode = "r+", bool fFlushOnClose = true) - : CDB(strFilename, pszMode, fFlushOnClose) + CWalletDB(const std::string &strFilename, const char *pszMode = "r+", bool fFlushOnCloseIn = true) + : CDB(strFilename, pszMode, fFlushOnCloseIn) { } From f9cec09008375dc06cbc0936af886b7acc3354f0 Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Wed, 16 Jan 2019 13:32:04 -0500 Subject: [PATCH 03/46] fix ECC_Start issue (#123) * fix ECC_Start issue * fix potential deadlock issue --- .gitignore | 1 + src/coins.cpp | 1 + src/key.cpp | 7 ++++--- src/test/coins_tests.cpp | 13 ++++++++++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 35a3c7a8..09ff2999 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ src/test/buildenv.py *.tar.gz *.exe +src/eccoind src/bitcoin src/bitcoind src/bitcoin-cli diff --git a/src/coins.cpp b/src/coins.cpp index 1a6d8361..b68f4f64 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -510,6 +510,7 @@ double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight, CAmount CCoinsViewCursor::~CCoinsViewCursor() {} static const size_t nMaxOutputsPerBlock = DEFAULT_LARGEST_TRANSACTION / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); + const Coin &AccessByTxid(const CCoinsViewCache &view, const uint256 &txid) { COutPoint iter(txid, 0); diff --git a/src/key.cpp b/src/key.cpp index 5583b60a..53556f51 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -442,9 +442,10 @@ void ECC_Start() { // Pass in a random blinding seed to the secp256k1 context. - std::vector > vseed(32); - GetRandBytes(vseed.data(), 32); - bool ret = secp256k1_context_randomize(ctx, vseed.data()); + uint8_t seed[32]; + LockObject(seed); + GetRandBytes(seed, 32); + bool ret = secp256k1_context_randomize(ctx, seed); assert(ret); } diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index f22d788f..62a3c62b 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -164,9 +164,16 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) { uint256 txid = txids[insecure_rand() % txids.size()]; // txid we're going to modify in this iteration. Coin &coin = result[COutPoint(txid, 0)]; - const Coin &entry = (insecure_rand() % 500 == 0) ? AccessByTxid(*stack.back(), txid) : - stack.back()->AccessCoin(COutPoint(txid, 0)); - BOOST_CHECK(coin == entry); + if ((insecure_rand() % 500) == 0) + { + const Coin &entry = AccessByTxid(*stack.back(), txid); + BOOST_CHECK(coin == entry); + } + else + { + const Coin &entry = stack.back()->AccessCoin(COutPoint(txid, 0)); + BOOST_CHECK(coin == entry); + } if (insecure_rand() % 5 == 0 || coin.IsSpent()) { From f58484d45bf0cd89689faea37cd227eeda109bdb Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Thu, 17 Jan 2019 10:51:24 -0500 Subject: [PATCH 04/46] [0.2.5.15] Cleanup connman (#116) * remove connman options and move startup around * remove network active toggle * reduce the amount of work around localnonce * change eviciton to use data activity instead of other metrics * keep track of connection failure attempts * get rid of message maker. less steps same result * make network_service_version easier to manage * remove unused banned list changed calls * use a threadgroup in connman, threadgroup takes in its interrupt varriable as argument * move network specific files into net folder * remove message thread specific interruptions * update formatted files --- src/.formatted-files | 29 +- src/Makefile.am | 29 +- src/blockgeneration/blockgeneration.cpp | 4 +- src/blockgeneration/miner.cpp | 2 +- src/chain/chainman.cpp | 2 +- src/crypto/scrypt.cpp | 2 +- src/crypto/scrypt.h | 2 +- src/eccoind.cpp | 3 +- src/httpserver.cpp | 2 +- src/init.cpp | 75 +- src/init.h | 6 - src/kernel.cpp | 2 +- src/main.cpp | 6 +- src/main.h | 2 +- src/{ => net}/addrdb.cpp | 2 +- src/{ => net}/addrdb.h | 0 src/{ => net}/addrman.cpp | 2 +- src/{ => net}/addrman.h | 4 +- src/{ => net}/messages.cpp | 176 ++-- src/{ => net}/messages.h | 6 +- src/{ => net}/net.cpp | 534 +++------- src/{ => net}/net.h | 1273 ++++++++++++----------- src/{ => net}/netaddress.cpp | 0 src/{ => net}/netaddress.h | 0 src/{ => net}/netbase.cpp | 2 +- src/{ => net}/netbase.h | 0 src/{ => net}/protocol.cpp | 2 +- src/{ => net}/protocol.h | 2 +- src/netmessagemaker.h | 50 - src/networks/networktemplate.h | 2 +- src/processblock.cpp | 5 +- src/processtx.cpp | 2 +- src/rpc/rpcmining.cpp | 2 +- src/rpc/rpcmisc.cpp | 4 +- src/rpc/rpcnet.cpp | 9 +- src/rpc/rpcrawtransaction.cpp | 2 +- src/rpc/rpcwallet.cpp | 4 +- src/test/DoS_tests.cpp | 2 +- src/test/addrman_tests.cpp | 2 +- src/test/exploit_tests.cpp | 4 +- src/test/net_tests.cpp | 6 +- src/test/netbase_tests.cpp | 2 +- src/test/rpc_tests.cpp | 4 +- src/test/test_bitcoin.cpp | 2 +- src/test/test_bitcoin_fuzzy.cpp | 6 +- src/threadgroup.h | 12 +- src/timedata.cpp | 2 +- src/torcontrol.cpp | 2 +- src/txdb.cpp | 3 +- src/version.h | 9 +- src/wallet/db.cpp | 4 +- src/wallet/wallet.cpp | 9 +- src/wallet/wallet.h | 2 +- src/wallet/walletdb.cpp | 2 +- 54 files changed, 1016 insertions(+), 1306 deletions(-) rename src/{ => net}/addrdb.cpp (99%) rename src/{ => net}/addrdb.h (100%) rename src/{ => net}/addrman.cpp (99%) rename src/{ => net}/addrman.h (99%) rename src/{ => net}/messages.cpp (94%) rename src/{ => net}/messages.h (97%) rename src/{ => net}/net.cpp (85%) rename src/{ => net}/net.h (84%) rename src/{ => net}/netaddress.cpp (100%) rename src/{ => net}/netaddress.h (100%) rename src/{ => net}/netbase.cpp (99%) rename src/{ => net}/netbase.h (100%) rename src/{ => net}/protocol.cpp (99%) rename src/{ => net}/protocol.h (99%) delete mode 100644 src/netmessagemaker.h diff --git a/src/.formatted-files b/src/.formatted-files index 7bdec713..ceb63e99 100644 --- a/src/.formatted-files +++ b/src/.formatted-files @@ -1,7 +1,3 @@ -addrdb.cpp -addrdb.h -addrman.cpp -addrman.h args.cpp args.h amount.cpp @@ -83,15 +79,20 @@ main.h memusage.h merkleblock.cpp merkleblock.h -messages.cpp -messages.h -net.cpp -net.h -netaddress.cpp -netaddress.h -netbase.cpp -netbase.h -netmessagemaker.h +net/addrdb.cpp +net/addrdb.h +net/addrman.cpp +net/addrman.h +net/messages.cpp +net/messages.h +net/net.cpp +net/net.h +net/netaddress.cpp +net/netaddress.h +net/netbase.cpp +net/netbase.h +net/protocol.cpp +net/protocol.h networks/netman.cpp networks/netman.h networks/network.h @@ -111,8 +112,6 @@ processheader.cpp processheader.h processtx.cpp processtx.h -protocol.cpp -protocol.h pubkey.cpp pubkey.h random.cpp diff --git a/src/Makefile.am b/src/Makefile.am index bc19f550..e9a5c121 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -38,8 +38,6 @@ endif .PHONY: FORCE check-symbols check-security check-formatting # bitcoin core # BITCOIN_CORE_H = \ - addrdb.h \ - addrman.h \ amount.h \ args.h \ arith_uint256.h \ @@ -81,13 +79,15 @@ BITCOIN_CORE_H = \ keystore.h \ limitedmap.h \ main.h \ - messages.h \ memusage.h \ merkleblock.h \ - net.h \ - netaddress.h \ - netbase.h \ - netmessagemaker.h \ + net/addrdb.h \ + net/addrman.h \ + net/messages.h \ + net/net.h \ + net/netaddress.h \ + net/netbase.h \ + net/protocol.h \ networks/netman.h \ networks/network.h \ networks/networktemplate.h \ @@ -102,7 +102,6 @@ BITCOIN_CORE_H = \ processblock.h \ processheader.h \ processtx.h \ - protocol.h \ pubkey.h \ random.h \ reverselock.h \ @@ -160,8 +159,8 @@ libbitcoin_server_a-clientversion.$(OBJEXT): build/build.h libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIC_FLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ - addrdb.cpp \ - addrman.cpp \ + net/addrdb.cpp \ + net/addrman.cpp \ blockgeneration/blockgeneration.cpp \ blockgeneration/miner.cpp \ blockgeneration/minter.cpp \ @@ -176,8 +175,8 @@ libbitcoin_server_a_SOURCES = \ kernel.cpp \ main.cpp \ merkleblock.cpp \ - net.cpp \ - netaddress.cpp \ + net/net.cpp \ + net/netaddress.cpp \ noui.cpp \ policy/fees.cpp \ policy/policy.cpp \ @@ -208,14 +207,14 @@ libbitcoin_server_a_SOURCES = \ crypto/hash.cpp \ key.cpp \ keystore.cpp \ - netbase.cpp \ + net/netbase.cpp \ chain/block.cpp \ chain/blockindex.cpp \ processblock.cpp \ processheader.cpp \ processtx.cpp \ chain/tx.cpp \ - protocol.cpp \ + net/protocol.cpp \ pubkey.cpp \ pbkdf2.cpp \ script/interpreter.cpp \ @@ -232,7 +231,7 @@ libbitcoin_server_a_SOURCES = \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ networks/netman.cpp \ - messages.cpp \ + net/messages.cpp \ random.cpp \ rpc/rpcprotocol.cpp \ support/cleanse.cpp \ diff --git a/src/blockgeneration/blockgeneration.cpp b/src/blockgeneration/blockgeneration.cpp index 67f02556..8e7664e3 100644 --- a/src/blockgeneration/blockgeneration.cpp +++ b/src/blockgeneration/blockgeneration.cpp @@ -82,7 +82,7 @@ void ThreadMiner(void *parg, bool shutdownOnly) { return; } - minerThreads = new thread_group(); + minerThreads = new thread_group(&shutdown_threads); CWallet *pwallet = (CWallet *)parg; try { @@ -116,7 +116,7 @@ void ThreadMinter(void *parg, bool shutdownOnly) { return; } - minterThreads = new thread_group(); + minterThreads = new thread_group(&shutdown_threads); CWallet *pwallet = (CWallet *)parg; try { diff --git a/src/blockgeneration/miner.cpp b/src/blockgeneration/miner.cpp index 0f59d094..2032b74a 100644 --- a/src/blockgeneration/miner.cpp +++ b/src/blockgeneration/miner.cpp @@ -27,7 +27,7 @@ #include "crypto/scrypt.h" #include "init.h" #include "kernel.h" -#include "net.h" +#include "net/net.h" #include "networks/netman.h" #include "networks/networktemplate.h" #include "policy/policy.h" diff --git a/src/chain/chainman.cpp b/src/chain/chainman.cpp index a42a1f50..95e05300 100644 --- a/src/chain/chainman.cpp +++ b/src/chain/chainman.cpp @@ -22,7 +22,7 @@ #include "init.h" #include "kernel.h" #include "main.h" -#include "messages.h" +#include "net/messages.h" #include "networks/netman.h" #include "processblock.h" #include "processheader.h" diff --git a/src/crypto/scrypt.cpp b/src/crypto/scrypt.cpp index 78a0a213..4e445649 100644 --- a/src/crypto/scrypt.cpp +++ b/src/crypto/scrypt.cpp @@ -33,7 +33,7 @@ #include "scrypt.h" #include "pbkdf2.h" -#include "net.h" +#include "net/net.h" #define SCRYPT_BUFFER_SIZE (131072 + 63) diff --git a/src/crypto/scrypt.h b/src/crypto/scrypt.h index d1d73e30..f5a20628 100644 --- a/src/crypto/scrypt.h +++ b/src/crypto/scrypt.h @@ -4,7 +4,7 @@ #include #include -#include "net.h" +#include "net/net.h" #include "chain/block.h" void *scrypt_buffer_alloc(); diff --git a/src/eccoind.cpp b/src/eccoind.cpp index 9d6b6ba4..67efdd03 100644 --- a/src/eccoind.cpp +++ b/src/eccoind.cpp @@ -55,7 +55,8 @@ void WaitForShutdown(thread_group *threadGroup) // bool AppInit(int argc, char *argv[]) { - thread_group threadGroup; + shutdown_threads.store(false); + thread_group threadGroup(&shutdown_threads); bool fRet = false; diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 29b4b4cd..4a0c22b2 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -23,7 +23,7 @@ #include "args.h" #include "init.h" -#include "netbase.h" +#include "net/netbase.h" #include "networks/netman.h" #include "rpc/rpcprotocol.h" // For HTTP status codes #include "sync.h" diff --git a/src/init.cpp b/src/init.cpp index 86934e30..bc5183ff 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -20,7 +20,6 @@ #include "init.h" -#include "addrman.h" #include "amount.h" #include "args.h" #include "blockgeneration/blockgeneration.h" @@ -32,8 +31,9 @@ #include "httpserver.h" #include "key.h" #include "main.h" -#include "messages.h" -#include "net.h" +#include "net/addrman.h" +#include "net/messages.h" +#include "net/net.h" #include "networks/netman.h" #include "networks/networktemplate.h" #include "policy/policy.h" @@ -899,15 +899,10 @@ void InitLogging() LogPrintf("Eccoin version %s (%s)\n", FormatFullVersion(), CLIENT_DATE); } -namespace -{ // Variables internal to initialization process only +int initMaxConnections; +int initUserMaxConnections; +int initFD; -ServiceFlags nRelevantServices = NODE_NETWORK; -int nMaxConnections; -int nUserMaxConnections; -int nFD; -ServiceFlags nLocalServices = NODE_NETWORK; -} // namespace void GenerateNetworkTemplates() { @@ -987,21 +982,24 @@ bool AppInit2(thread_group &threadGroup) // Make sure enough file descriptors are available int nBind = std::max((int)gArgs.IsArgSet("-bind") + (int)gArgs.IsArgSet("-whitebind"), 1); int nUserMaxConnections = gArgs.GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); - nMaxConnections = std::max(nUserMaxConnections, 0); + initMaxConnections = std::max(nUserMaxConnections, 0); // Trim requested connection counts, to fit into system limitations - nMaxConnections = std::max(std::min(nMaxConnections, (int)(FD_SETSIZE - nBind - MIN_CORE_FILEDESCRIPTORS)), 0); - int nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS); - if (nFD < MIN_CORE_FILEDESCRIPTORS) + initMaxConnections = + std::max(std::min(initMaxConnections, (int)(FD_SETSIZE - nBind - MIN_CORE_FILEDESCRIPTORS)), 0); + initFD = RaiseFileDescriptorLimit(initMaxConnections + MIN_CORE_FILEDESCRIPTORS); + if (initFD < MIN_CORE_FILEDESCRIPTORS) + { return InitError(_("Not enough file descriptors available.")); - nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS, nMaxConnections); + } + initMaxConnections = std::min(initFD - MIN_CORE_FILEDESCRIPTORS, initMaxConnections); - if (nMaxConnections < nUserMaxConnections) + if (initMaxConnections < nUserMaxConnections) { LogPrintf("Reducing -maxconnections from %d to %d, because of system limitations.", nUserMaxConnections, - nMaxConnections); + initMaxConnections); InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), - nUserMaxConnections, nMaxConnections)); + nUserMaxConnections, initMaxConnections)); } // ********************************************************* Step 3: parameter-to-internal-flags @@ -1159,9 +1157,6 @@ bool AppInit2(thread_group &threadGroup) // Option to startup with mocktime set (used for regression testing): SetMockTime(gArgs.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op - if (gArgs.GetBoolArg("-peerbloomfilters", true)) - nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); - fEnableReplacement = gArgs.GetBoolArg("-mempoolreplacement", DEFAULT_ENABLE_REPLACEMENT); if ((!fEnableReplacement) && gArgs.IsArgSet("-mempoolreplacement")) { @@ -1225,7 +1220,7 @@ bool AppInit2(thread_group &threadGroup) LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); LogPrintf("Using data directory %s\n", strDataDir); LogPrintf("Using config file %s\n", gArgs.GetConfigFile().string()); - LogPrintf("Using at most %i connections (%i file descriptors available)\n", nMaxConnections, nFD); + LogPrintf("Using at most %i connections (%i file descriptors available)\n", initMaxConnections, initFD); std::ostringstream strErrors; LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); @@ -1257,7 +1252,7 @@ bool AppInit2(thread_group &threadGroup) if (!fDisableWallet) { LogPrintf("Using wallet %s\n", strWalletFile); - uiInterface.InitMessage(_("Verifying wallet...")); + LogPrintf("Verifying wallet..."); std::string warningString; std::string errorString; @@ -1439,14 +1434,6 @@ bool AppInit2(thread_group &threadGroup) } } - uint64_t nMaxOutboundLimit = 0; - uint64_t nMaxOutboundTimeframe = MAX_UPLOAD_TIMEFRAME; - - if (gArgs.IsArgSet("-maxuploadtarget")) - { - nMaxOutboundLimit = gArgs.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET) * 1024 * 1024; - } - // ********************************************************* Step 7: load block chain fReindex = gArgs.GetBoolArg("-reindex", false); @@ -1506,7 +1493,6 @@ bool AppInit2(thread_group &threadGroup) bool fReset = fReindex; std::string strLoadError; - uiInterface.InitMessage(_("Loading block index...")); LogPrintf("Loading block index..."); nStart = GetTimeMillis(); do @@ -1574,8 +1560,6 @@ bool AppInit2(thread_group &threadGroup) } // verify the blocks - - uiInterface.InitMessage(_("Verifying blocks...")); LogPrintf("Verifying blocks..."); { @@ -1672,7 +1656,7 @@ bool AppInit2(thread_group &threadGroup) if (gArgs.IsArgSet("-blocknotify")) uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); - uiInterface.InitMessage(_("Activating best chain...")); + LogPrintf("Activating best chain..."); std::vector vImportFiles; if (gArgs.IsArgSet("-loadblock")) @@ -1718,22 +1702,7 @@ bool AppInit2(thread_group &threadGroup) MapPort(gArgs.GetBoolArg("-upnp", DEFAULT_UPNP)); std::string strNodeError; - CConnman::Options connOptions; - connOptions.nLocalServices = nLocalServices; - connOptions.nRelevantServices = nRelevantServices; - connOptions.nMaxConnections = nMaxConnections; - connOptions.nMaxOutbound = std::min(MAX_OUTBOUND_CONNECTIONS, connOptions.nMaxConnections); - connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; - connOptions.nMaxFeeler = 1; - connOptions.nBestHeight = pnetMan->getChainActive()->chainActive.Height(); - connOptions.uiInterface = &uiInterface; - connOptions.nSendBufferMaxSize = 1000 * gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); - connOptions.nReceiveFloodSize = 1000 * gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); - - connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; - connOptions.nMaxOutboundLimit = nMaxOutboundLimit; - - if (!connman.Start(strNodeError, connOptions)) + if (!connman.Start(strNodeError)) { return InitError(strNodeError); } @@ -1751,7 +1720,7 @@ bool AppInit2(thread_group &threadGroup) // ********************************************************* Step 12: finished SetRPCWarmupFinished(); - uiInterface.InitMessage(_("Done loading")); + LogPrintf("Done loading"); if (pwalletMain) diff --git a/src/init.h b/src/init.h index dbe47d74..e408b000 100644 --- a/src/init.h +++ b/src/init.h @@ -26,14 +26,8 @@ #include "wallet/wallet.h" #include -class CScheduler; class CWallet; -namespace boost -{ -class thread_group; -} // namespace boost - extern CWallet *pwalletMain; extern CNetworkManager *pnetMan; diff --git a/src/kernel.cpp b/src/kernel.cpp index 9be9bc48..4281bb80 100644 --- a/src/kernel.cpp +++ b/src/kernel.cpp @@ -26,7 +26,7 @@ #include "init.h" #include "kernel.h" #include "main.h" -#include "net.h" +#include "net/net.h" #include "networks/netman.h" #include "networks/networktemplate.h" #include "script/stakescript.h" diff --git a/src/main.cpp b/src/main.cpp index 7302f6dc..3289b26b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -20,7 +20,6 @@ #include "main.h" -#include "addrman.h" #include "args.h" #include "arith_uint256.h" #include "chain/chain.h" @@ -33,8 +32,9 @@ #include "init.h" #include "kernel.h" #include "merkleblock.h" -#include "messages.h" -#include "net.h" +#include "net/addrman.h" +#include "net/messages.h" +#include "net/net.h" #include "networks/netman.h" #include "networks/networktemplate.h" #include "policy/policy.h" diff --git a/src/main.h b/src/main.h index b344a717..029e9d3d 100644 --- a/src/main.h +++ b/src/main.h @@ -24,7 +24,7 @@ #include "amount.h" #include "chain/chain.h" #include "coins.h" -#include "net.h" +#include "net/net.h" #include "script/script_error.h" #include "sync.h" #include "uint256.h" diff --git a/src/addrdb.cpp b/src/net/addrdb.cpp similarity index 99% rename from src/addrdb.cpp rename to src/net/addrdb.cpp index 0c2a4eee..f04316e3 100644 --- a/src/addrdb.cpp +++ b/src/net/addrdb.cpp @@ -20,10 +20,10 @@ #include "addrdb.h" -#include "addrman.h" #include "clientversion.h" #include "crypto/hash.h" #include "init.h" +#include "net/addrman.h" #include "random.h" #include "streams.h" #include "tinyformat.h" diff --git a/src/addrdb.h b/src/net/addrdb.h similarity index 100% rename from src/addrdb.h rename to src/net/addrdb.h diff --git a/src/addrman.cpp b/src/net/addrman.cpp similarity index 99% rename from src/addrman.cpp rename to src/net/addrman.cpp index 2232cdbb..c158f868 100644 --- a/src/addrman.cpp +++ b/src/net/addrman.cpp @@ -17,7 +17,7 @@ * along with this program. If not, see . */ -#include "addrman.h" +#include "net/addrman.h" #include "crypto/hash.h" #include "serialize.h" diff --git a/src/addrman.h b/src/net/addrman.h similarity index 99% rename from src/addrman.h rename to src/net/addrman.h index 23b637a7..fc1663a6 100644 --- a/src/addrman.h +++ b/src/net/addrman.h @@ -19,8 +19,8 @@ #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include "netbase.h" -#include "protocol.h" +#include "net/netbase.h" +#include "net/protocol.h" #include "random.h" #include "sync.h" #include "timedata.h" diff --git a/src/messages.cpp b/src/net/messages.cpp similarity index 94% rename from src/messages.cpp rename to src/net/messages.cpp index 326f34b0..163f6f59 100644 --- a/src/messages.cpp +++ b/src/net/messages.cpp @@ -18,9 +18,8 @@ * along with this program. If not, see . */ -#include "messages.h" +#include "net/messages.h" -#include "addrman.h" #include "args.h" #include "chain/chain.h" #include "chain/tx.h" @@ -28,7 +27,8 @@ #include "init.h" #include "main.h" #include "merkleblock.h" -#include "netmessagemaker.h" +#include "net/addrman.h" +#include "net/protocol.h" #include "networks/netman.h" #include "networks/networktemplate.h" #include "policy/fees.h" @@ -36,7 +36,6 @@ #include "processblock.h" #include "processheader.h" #include "processtx.h" -#include "protocol.h" #include "serialize.h" #include "sync.h" #include "txmempool.h" @@ -94,6 +93,9 @@ static uint256 most_recent_block_hash; /** Map maintaining per-node state. Requires cs_main. */ std::map mapNodeState; +uint64_t nLocalHostNonce = 0; +extern CCriticalSection cs_mapInboundConnectionTracker; +extern std::map mapInboundConnectionTracker; // Requires cs_main. CNodeState *State(NodeId pnode) @@ -116,7 +118,6 @@ uint32_t GetFetchFlags(CNode *pfrom, const CBlockIndex *pprev, const Consensus:: void PushNodeVersion(CNode *pnode, CConnman &connman, int64_t nTime) { ServiceFlags nLocalNodeServices = pnode->GetLocalServices(); - uint64_t nonce = pnode->GetLocalNonce(); int nNodeStartingHeight = pnode->GetMyStartingHeight(); NodeId nodeid = pnode->GetId(); CAddress addr = pnode->addr; @@ -124,9 +125,10 @@ void PushNodeVersion(CNode *pnode, CConnman &connman, int64_t nTime) CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); CAddress addrMe = CAddress(CService(), nLocalNodeServices); - connman.PushMessage(pnode, CNetMsgMaker(MIN_PROTO_VERSION) - .Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, - addrYou, addrMe, nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes)); + GetRandBytes((unsigned char *)&nLocalHostNonce, sizeof(nLocalHostNonce)); + + connman.PushMessage(pnode, NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, + addrMe, nLocalHostNonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes); if (fLogIPs) { @@ -681,9 +683,8 @@ static bool SendRejectsAndCheckIfBanned(CNode *pnode, CConnman &connman) for (const CBlockReject &reject : state.rejects) { - connman.PushMessage(pnode, CNetMsgMaker(MIN_PROTO_VERSION) - .Make(NetMsgType::REJECT, std::string(NetMsgType::BLOCK), reject.chRejectCode, - reject.strRejectReason, reject.hashBlock)); + connman.PushMessage(pnode, NetMsgType::REJECT, std::string(NetMsgType::BLOCK), reject.chRejectCode, + reject.strRejectReason, reject.hashBlock); } state.rejects.clear(); @@ -780,7 +781,6 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr &pb void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr &pblock) { - const CNetMsgMaker msgMaker(PROTOCOL_VERSION); LOCK(cs_main); static int nHighestFastAnnounce = 0; if (pindex->nHeight <= nHighestFastAnnounce) @@ -794,7 +794,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: most_recent_block_hash = hashBlock; most_recent_block = pblock; } - connman->ForEachNode([this, pindex, &msgMaker, &hashBlock](CNode *pnode) { + connman->ForEachNode([this, pindex, &hashBlock](CNode *pnode) { // TODO: Avoid the repeated-serialization here if (pnode->fDisconnect) { @@ -810,7 +810,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: hashBlock.ToString(), pnode->id); std::vector vHeaders; vHeaders.push_back(pindex->GetBlockHeader()); - connman->PushMessage(pnode, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman->PushMessage(pnode, NetMsgType::HEADERS, vHeaders); state.pindexBestHeaderSent = pindex; } }); @@ -851,7 +851,6 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, } } }); - connman->WakeMessageHandler(); } nTimeBestReceived = GetTime(); @@ -913,12 +912,8 @@ bool AlreadyHave(const CInv &inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) return true; } -void static ProcessGetData(CNode *pfrom, - CConnman &connman, - const Consensus::Params &consensusParams, - const std::atomic &interruptMsgProc) +void static ProcessGetData(CNode *pfrom, CConnman &connman, const Consensus::Params &consensusParams) { - const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); std::deque::iterator it = pfrom->vRecvGetData.begin(); std::vector vNotFound; @@ -934,11 +929,6 @@ void static ProcessGetData(CNode *pfrom, const CInv &inv = *it; - if (interruptMsgProc) - { - return; - } - it++; if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) @@ -1000,7 +990,7 @@ void static ProcessGetData(CNode *pfrom, } if (inv.type == MSG_BLOCK) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, NetMsgType::BLOCK, block); } else if (inv.type == MSG_FILTERED_BLOCK) { @@ -1016,7 +1006,7 @@ void static ProcessGetData(CNode *pfrom, } if (sendMerkleBlock) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + connman.PushMessage(pfrom, NetMsgType::MERKLEBLOCK, merkleBlock); // CMerkleBlock just contains hashes, so also push // any transactions in the block the client did not // see. This avoids hurting performance by @@ -1031,7 +1021,7 @@ void static ProcessGetData(CNode *pfrom, typedef std::pair PairType; for (PairType &pair : merkleBlock.vMatchedTxn) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, block.vtx[pair.first])); + connman.PushMessage(pfrom, NetMsgType::TX, block.vtx[pair.first]); } } // else @@ -1047,7 +1037,7 @@ void static ProcessGetData(CNode *pfrom, // so they don't wait for other stuff first. std::vector vInv; vInv.push_back(CInv(MSG_BLOCK, pnetMan->getChainActive()->chainActive.Tip()->GetBlockHash())); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + connman.PushMessage(pfrom, NetMsgType::INV, vInv); pfrom->hashContinue.SetNull(); } } @@ -1061,7 +1051,7 @@ void static ProcessGetData(CNode *pfrom, int nSendFlags = 0; if (mi != mapRelay.end()) { - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, mi->second)); + connman.PushMessage(pfrom, NetMsgType::TX, mi->second); push = true; } if (!push) @@ -1088,7 +1078,7 @@ void static ProcessGetData(CNode *pfrom, // about (and store and rebroadcast and risk analyze) the dependencies // of transactions relevant to them, without having to download the // entire memory pool. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + connman.PushMessage(pfrom, NetMsgType::NOTFOUND, vNotFound); } } @@ -1113,8 +1103,7 @@ bool static ProcessMessage(CNode *pfrom, std::string strCommand, CDataStream &vRecv, int64_t nTimeReceived, - CConnman &connman, - const std::atomic &interruptMsgProc) + CConnman &connman) { const CNetworkTemplate &chainparams = pnetMan->getActivePaymentNetwork(); RandAddSeedPerfmon(); @@ -1148,9 +1137,8 @@ bool static ProcessMessage(CNode *pfrom, // Each connection can only send one version message if (pfrom->nVersion != 0) { - connman.PushMessage(pfrom, - CNetMsgMaker(MIN_PROTO_VERSION) - .Make(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, std::string("Duplicate version message"))); + connman.PushMessage( + pfrom, NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, std::string("Duplicate version message")); LOCK(cs_main); Misbehaving(pfrom, 1, "multiple-version"); return false; @@ -1180,9 +1168,8 @@ bool static ProcessMessage(CNode *pfrom, { LogPrintf("peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, nServices, pfrom->nServicesExpected); - connman.PushMessage(pfrom, CNetMsgMaker(MIN_PROTO_VERSION) - .Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, - strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); + connman.PushMessage(pfrom, NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + strprintf("Expected to offer services %08x", pfrom->nServicesExpected)); pfrom->fDisconnect = true; return false; } @@ -1191,9 +1178,8 @@ bool static ProcessMessage(CNode *pfrom, { // disconnect from peers older than this proto version LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, nVersion); - connman.PushMessage(pfrom, CNetMsgMaker(MIN_PROTO_VERSION) - .Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, - strprintf("Version must be %d or greater", MIN_PROTO_VERSION))); + connman.PushMessage(pfrom, NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, + strprintf("Version must be %d or greater", MIN_PROTO_VERSION)); pfrom->fDisconnect = true; return false; } @@ -1216,7 +1202,7 @@ bool static ProcessMessage(CNode *pfrom, vRecv >> fRelay; } // Disconnect if we connected to ourself - if (pfrom->fInbound && !connman.CheckIncomingNonce(nNonce)) + if (nNonce == nLocalHostNonce && nNonce > 1) { LogPrintf("connected to self at %s, disconnecting\n", pfrom->addr.ToString()); pfrom->fDisconnect = true; @@ -1234,7 +1220,7 @@ bool static ProcessMessage(CNode *pfrom, PushNodeVersion(pfrom, connman, GetAdjustedTime()); } - connman.PushMessage(pfrom, CNetMsgMaker(MIN_PROTO_VERSION).Make(NetMsgType::VERACK)); + connman.PushMessage(pfrom, NetMsgType::VERACK); pfrom->nServices = nServices; pfrom->SetAddrLocal(addrMe); @@ -1284,7 +1270,7 @@ bool static ProcessMessage(CNode *pfrom, // Get recent addresses if (pfrom->fOneShot || connman.GetAddressCount() < 1000) { - connman.PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + connman.PushMessage(pfrom, NetMsgType::GETADDR); pfrom->fGetAddr = true; } connman.MarkAddressGood(pfrom->addr); @@ -1323,8 +1309,6 @@ bool static ProcessMessage(CNode *pfrom, return false; } - const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - if (strCommand == NetMsgType::VERACK) { @@ -1344,19 +1328,27 @@ bool static ProcessMessage(CNode *pfrom, // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + connman.PushMessage(pfrom, NetMsgType::SENDHEADERS); } pfrom->fSuccessfullyConnected = true; } - /* - else if (!pfrom->fSuccessfullyConnected) { + else if (!pfrom->fSuccessfullyConnected) + { + { // Must have a verack message before anything else LOCK(cs_main); Misbehaving(pfrom, 1, "missing-verack"); - return false; } - */ + { + // update connection tracker which is used by the connection slot algorithm. + LOCK(cs_mapInboundConnectionTracker); + CNetAddr ipAddress = (CNetAddr)pfrom->addr; + mapInboundConnectionTracker[ipAddress].nEvictions += 1; + mapInboundConnectionTracker[ipAddress].nLastEvictionTime = GetTime(); + } + return false; + } else if (strCommand == NetMsgType::ADDR) { @@ -1452,11 +1444,6 @@ bool static ProcessMessage(CNode *pfrom, { CInv &inv = vInv[nInv]; - if (interruptMsgProc) - { - return true; - } - bool fAlreadyHave = AlreadyHave(inv); LogPrintf("got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); @@ -1476,10 +1463,9 @@ bool static ProcessMessage(CNode *pfrom, // should get the headers for first, we now only provide a // getheaders response here. When we receive the headers, we // will then ask for the blocks we need. - connman.PushMessage( - pfrom, msgMaker.Make(NetMsgType::GETHEADERS, pnetMan->getChainActive()->chainActive.GetLocator( - pnetMan->getChainActive()->pindexBestHeader), - inv.hash)); + connman.PushMessage(pfrom, NetMsgType::GETHEADERS, + pnetMan->getChainActive()->chainActive.GetLocator(pnetMan->getChainActive()->pindexBestHeader), + inv.hash); CNodeState *nodestate = State(pfrom->GetId()); if (CanDirectFetch(chainparams.GetConsensus()) && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) @@ -1514,7 +1500,7 @@ bool static ProcessMessage(CNode *pfrom, if (!vToFetch.empty()) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vToFetch)); + connman.PushMessage(pfrom, NetMsgType::GETDATA, vToFetch); } } @@ -1538,7 +1524,7 @@ bool static ProcessMessage(CNode *pfrom, } pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); - ProcessGetData(pfrom, connman, chainparams.GetConsensus(), interruptMsgProc); + ProcessGetData(pfrom, connman, chainparams.GetConsensus()); } @@ -1662,7 +1648,7 @@ bool static ProcessMessage(CNode *pfrom, // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : pnetMan->getChainActive()->chainActive.Tip(); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman.PushMessage(pfrom, NetMsgType::HEADERS, vHeaders); } else if (strCommand == NetMsgType::TX) @@ -1873,8 +1859,8 @@ bool static ProcessMessage(CNode *pfrom, // Never send AcceptToMemoryPool's internal codes over P2P. if (state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, uint8_t(state.GetRejectCode()), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); + connman.PushMessage(pfrom, NetMsgType::REJECT, strCommand, uint8_t(state.GetRejectCode()), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); } if (nDoS > 0) { @@ -1943,8 +1929,8 @@ bool static ProcessMessage(CNode *pfrom, // from there instead. LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, - pnetMan->getChainActive()->chainActive.GetLocator(pindexLast), uint256())); + connman.PushMessage(pfrom, NetMsgType::GETHEADERS, + pnetMan->getChainActive()->chainActive.GetLocator(pindexLast), uint256()); } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); @@ -1999,7 +1985,7 @@ bool static ProcessMessage(CNode *pfrom, } if (vGetData.size() > 0) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman.PushMessage(pfrom, NetMsgType::GETDATA, vGetData); } } } @@ -2035,9 +2021,8 @@ bool static ProcessMessage(CNode *pfrom, if (state.IsInvalid(nDoS)) { assert(state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes - connman.PushMessage( - pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash)); + connman.PushMessage(pfrom, NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash); if (nDoS > 0) { LOCK(cs_main); @@ -2104,13 +2089,13 @@ bool static ProcessMessage(CNode *pfrom, vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + connman.PushMessage(pfrom, NetMsgType::INV, vInv); vInv.clear(); } } if (vInv.size() > 0) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + connman.PushMessage(pfrom, NetMsgType::INV, vInv); } } @@ -2131,7 +2116,7 @@ bool static ProcessMessage(CNode *pfrom, // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + connman.PushMessage(pfrom, NetMsgType::PONG, nonce); } } @@ -2294,7 +2279,7 @@ bool static ProcessMessage(CNode *pfrom, return true; } -bool ProcessMessages(CNode *pfrom, CConnman &connman, const std::atomic &interruptMsgProc) +bool ProcessMessages(CNode *pfrom, CConnman &connman) { // // Message format @@ -2308,7 +2293,7 @@ bool ProcessMessages(CNode *pfrom, CConnman &connman, const std::atomic &i if (!pfrom->vRecvGetData.empty()) { - ProcessGetData(pfrom, connman, pnetMan->getActivePaymentNetwork()->GetConsensus(), interruptMsgProc); + ProcessGetData(pfrom, connman, pnetMan->getActivePaymentNetwork()->GetConsensus()); } if (pfrom->fDisconnect) @@ -2381,11 +2366,7 @@ bool ProcessMessages(CNode *pfrom, CConnman &connman, const std::atomic &i bool fRet = false; try { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, connman, interruptMsgProc); - if (interruptMsgProc) - { - return false; - } + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, connman); if (!pfrom->vRecvGetData.empty()) { fMoreWork = true; @@ -2394,8 +2375,7 @@ bool ProcessMessages(CNode *pfrom, CConnman &connman, const std::atomic &i catch (const std::ios_base::failure &e) { connman.PushMessage( - pfrom, CNetMsgMaker(MIN_PROTO_VERSION) - .Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); + pfrom, NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message")); if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv @@ -2440,7 +2420,7 @@ bool ProcessMessages(CNode *pfrom, CConnman &connman, const std::atomic &i return fMoreWork; } -bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interruptMsgProc) +bool SendMessages(CNode *pto, CConnman &connman) { const Consensus::Params &consensusParams = pnetMan->getActivePaymentNetwork()->GetConsensus(); @@ -2450,10 +2430,6 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr return true; } - // If we get here, the outgoing message serialization version is set and - // can't change. - const CNetMsgMaker msgMaker(pto->GetSendVersion()); - // // Message: ping // @@ -2478,7 +2454,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr pto->fPingQueued = false; pto->nPingUsecStart = GetTimeMicros(); pto->nPingNonceSent = nonce; - connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); + connman.PushMessage(pto, NetMsgType::PING, nonce); } // Acquire cs_main for IsInitialBlockDownload() and CNodeState() @@ -2519,7 +2495,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + connman.PushMessage(pto, NetMsgType::ADDR, vAddr); vAddr.clear(); } } @@ -2527,7 +2503,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr pto->vAddrToSend.clear(); if (!vAddr.empty()) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + connman.PushMessage(pto, NetMsgType::ADDR, vAddr); } // we only send the big addr message once @@ -2572,8 +2548,8 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, - pnetMan->getChainActive()->chainActive.GetLocator(pindexStart), uint256())); + connman.PushMessage( + pto, NetMsgType::GETHEADERS, pnetMan->getChainActive()->chainActive.GetLocator(pindexStart), uint256()); } } @@ -2676,7 +2652,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->id); } - connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman.PushMessage(pto, NetMsgType::HEADERS, vHeaders); state.pindexBestHeaderSent = pBestIndex; } else @@ -2730,7 +2706,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr vInv.push_back(CInv(MSG_BLOCK, hash)); if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman.PushMessage(pto, NetMsgType::INV, vInv); vInv.clear(); } } @@ -2794,7 +2770,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr nRelayedTransactions++; if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman.PushMessage(pto, NetMsgType::INV, vInv); vInv.clear(); } pto->filterInventoryKnown.insert(hash); @@ -2803,7 +2779,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr } if (!vInv.empty()) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman.PushMessage(pto, NetMsgType::INV, vInv); } // Detect whether we're stalling @@ -2887,7 +2863,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr vGetData.push_back(inv); if (vGetData.size() >= 1000) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman.PushMessage(pto, NetMsgType::GETDATA, vGetData); vGetData.clear(); } } @@ -2900,7 +2876,7 @@ bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interr } if (!vGetData.empty()) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman.PushMessage(pto, NetMsgType::GETDATA, vGetData); } return true; } diff --git a/src/messages.h b/src/net/messages.h similarity index 97% rename from src/messages.h rename to src/net/messages.h index 8f7dbca2..219f8f4c 100644 --- a/src/messages.h +++ b/src/net/messages.h @@ -22,7 +22,7 @@ #define MESSAGES_H #include "chain/blockindex.h" -#include "net.h" +#include "net/net.h" #include "validationinterface.h" @@ -193,10 +193,10 @@ void Misbehaving(NodeId nodeid, int howmuch, const std::string &reason); /** Process protocol messages received from a given node */ -bool ProcessMessages(CNode *pfrom, CConnman &connman, const std::atomic &interrupt); +bool ProcessMessages(CNode *pfrom, CConnman &connman); /**Send queued protocol messages to be sent to a give node. */ -bool SendMessages(CNode *pto, CConnman &connman, const std::atomic &interrupt); +bool SendMessages(CNode *pto, CConnman &connman); /** Returns a bool indicating whether we requested this block. If we did request it, marks it as receieved and removes * block from in flight list*/ diff --git a/src/net.cpp b/src/net/net.cpp similarity index 85% rename from src/net.cpp rename to src/net/net.cpp index f01f4daf..c1382c72 100644 --- a/src/net.cpp +++ b/src/net/net.cpp @@ -18,16 +18,16 @@ * along with this program. If not, see . */ -#include "net.h" -#include "args.h" +#include "net/net.h" -#include "addrman.h" +#include "args.h" #include "chain/tx.h" #include "clientversion.h" #include "consensus/consensus.h" #include "crypto/common.h" #include "crypto/hash.h" #include "init.h" +#include "net/addrman.h" #include "networks/netman.h" #include "ui_interface.h" #include "util/utilstrencodings.h" @@ -94,6 +94,10 @@ limitedmap mapAlreadyAskedFor(MAX_INV_SZ); std::string strSubVersion; +// Connection Slot mitigation - used to determine how many connection attempts over time +CCriticalSection cs_mapInboundConnectionTracker; +std::map mapInboundConnectionTracker; + // Signals for message handling static CNodeSignals g_signals; CNodeSignals &GetNodeSignals() { return g_signals; } @@ -335,17 +339,6 @@ CNode *CConnman::FindNode(const CService &addr) return nullptr; } -bool CConnman::CheckIncomingNonce(uint64_t nonce) -{ - LOCK(cs_vNodes); - for (CNode *pnode : vNodes) - { - if (!pnode->fSuccessfullyConnected && !pnode->fInbound && pnode->GetLocalNonce() == nonce) - return false; - } - return true; -} - CNode *CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure) { if (pszDest == nullptr) @@ -467,10 +460,6 @@ void CConnman::ClearBanned() // Store banlist to disk. DumpBanlist(); - if (clientInterface) - { - clientInterface->BannedListChanged(); - } } bool CConnman::IsBanned(CNetAddr ip) @@ -540,11 +529,6 @@ void CConnman::Ban(const CSubNet &subNet, const BanReason &banReason, int64_t ba } } - if (clientInterface) - { - clientInterface->BannedListChanged(); - } - { LOCK(cs_vNodes); for (CNode *pnode : vNodes) @@ -579,12 +563,6 @@ bool CConnman::Unban(const CSubNet &subNet) } setBannedIsDirty = true; } - - if (clientInterface) - { - clientInterface->BannedListChanged(); - } - // Store banlist to disk immediately. DumpBanlist(); return true; @@ -810,6 +788,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool &complete { i = mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER); } + nActivityBytes += msg.hdr.nMessageSize; assert(i != mapRecvBytesPerMsgCmd.end()); i->second += msg.hdr.nMessageSize + CMessageHeader::HEADER_SIZE; @@ -822,25 +801,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool &complete return true; } -void CNode::SetSendVersion(int nVersionIn) -{ - // Send version may only be changed in the version message, and only one - // version message is allowed per session. We can therefore treat this value - // as const and even atomic as long as it's only used once a version message - // has been successfully processed. Any attempt to set this twice is an - // error. - if (nSendVersion != 0) - { - error("Send version already set for node: %i. Refusing to change from " - "%i to %i", - id, nSendVersion, nVersionIn); - } - else - { - nSendVersion = nVersionIn; - } -} - +void CNode::SetSendVersion(int nVersionIn) { nSendVersion = nVersionIn; } int CNode::GetSendVersion() const { // The send version should always be explicitly set to INIT_PROTO_VERSION @@ -989,200 +950,84 @@ size_t CConnman::SocketSendData(CNode *pnode) const return nSentSize; } -struct NodeEvictionCandidate -{ - NodeId id; - int64_t nTimeConnected; - int64_t nMinPingUsecTime; - int64_t nLastBlockTime; - int64_t nLastTXTime; - bool fRelevantServices; - bool fRelayTxes; - bool fBloomFilter; - CAddress addr; - uint64_t nKeyedNetGroup; -}; - -static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - return a.nMinPingUsecTime > b.nMinPingUsecTime; -} - -static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - return a.nTimeConnected > b.nTimeConnected; -} - -static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - return a.nKeyedNetGroup < b.nKeyedNetGroup; -} - -static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - // There is a fall-through here because it is common for a node to have many - // peers which have not yet relayed a block. - if (a.nLastBlockTime != b.nLastBlockTime) - { - return a.nLastBlockTime < b.nLastBlockTime; - } - - if (a.fRelevantServices != b.fRelevantServices) - { - return b.fRelevantServices; - } - - return a.nTimeConnected > b.nTimeConnected; -} - -static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +static bool CompareNodeActivityBytes(const CNodeRef &a, const CNodeRef &b) { - // There is a fall-through here because it is common for a node to have more - // than a few peers that have not yet relayed txn. - if (a.nLastTXTime != b.nLastTXTime) - { - return a.nLastTXTime < b.nLastTXTime; - } - - if (a.fRelayTxes != b.fRelayTxes) - { - return b.fRelayTxes; - } - - if (a.fBloomFilter != b.fBloomFilter) - { - return a.fBloomFilter; - } - - return a.nTimeConnected > b.nTimeConnected; + return a->nActivityBytes < b->nActivityBytes; } -/** - * Try to find a connection to evict when the node is full. Extreme care must be - * taken to avoid opening the node to attacker triggered network partitioning. - * The strategy used here is to protect a small number of peers for each of - * several distinct characteristics which are difficult to forge. In order to - * partition a node the attacker must be simultaneously better at all of them - * than honest peers. - */ bool CConnman::AttemptToEvictConnection() { - std::vector vEvictionCandidates; + std::vector vEvictionCandidates; + std::vector vEvictionCandidatesByActivity; { LOCK(cs_vNodes); - + static int64_t nLastTime = GetTime(); for (CNode *node : vNodes) { + int64_t nNow = GetTime(); + node->nActivityBytes *= pow(1.0 - 1.0 / 7200, (double)(nNow - nLastTime)); // exponential 2 hour decay + if (node->fWhitelisted || !node->fInbound || node->fDisconnect) { continue; } - NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, - node->nLastBlockTime, node->nLastTXTime, (node->nServices & nRelevantServices) == nRelevantServices, - node->fRelayTxes, node->pfilter != nullptr, node->addr, node->nKeyedNetGroup}; - vEvictionCandidates.push_back(candidate); + vEvictionCandidates.push_back(CNodeRef(node)); } + nLastTime = GetTime(); } + vEvictionCandidatesByActivity = vEvictionCandidates; if (vEvictionCandidates.empty()) { return false; } - // Protect connections with certain characteristics + // If we get here then we prioritize connections based on activity. The least active incoming peer is + // de-prioritized based on bytes in and bytes out. A whitelisted peer will always get a connection and there is + // no need here to check whether the peer is whitelisted or not. + std::sort(vEvictionCandidatesByActivity.begin(), vEvictionCandidatesByActivity.end(), CompareNodeActivityBytes); + vEvictionCandidatesByActivity[0]->fDisconnect = true; - // Deterministically select 4 peers to protect by netgroup. An attacker - // cannot predict which netgroups will be protected. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNetGroupKeyed); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast(vEvictionCandidates.size())), - vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) - { - return false; - } - - // Protect the 8 nodes with the lowest minimum ping time. An attacker cannot - // manipulate this metric without physically moving nodes closer to the - // target. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeMinPingTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(8, static_cast(vEvictionCandidates.size())), - vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) + // BU - update the connection tracker { - return false; - } - - // Protect 4 nodes that most recently sent us transactions. An attacker - // cannot manipulate this metric without performing useful work. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeTXTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast(vEvictionCandidates.size())), - vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) - { - return false; - } - - // Protect 4 nodes that most recently sent us blocks. An attacker cannot - // manipulate this metric without performing useful work. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeBlockTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast(vEvictionCandidates.size())), - vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) - { - return false; - } - - // Protect the half of the remaining nodes which have been connected the - // longest. This replicates the non-eviction implicit behavior, and - // precludes attacks that start later. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeTimeConnected); - vEvictionCandidates.erase( - vEvictionCandidates.end() - static_cast(vEvictionCandidates.size() / 2), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) - { - return false; - } + double nEvictions = 0; + LOCK(cs_mapInboundConnectionTracker); + CNetAddr ipAddress = (CNetAddr)vEvictionCandidatesByActivity[0]->addr; + if (mapInboundConnectionTracker.count(ipAddress)) + { + // Decay the current number of evictions (over 1800 seconds) depending on the last eviction + int64_t nTimeElapsed = GetTime() - mapInboundConnectionTracker[ipAddress].nLastEvictionTime; + double nRatioElapsed = (double)nTimeElapsed / 1800; + nEvictions = mapInboundConnectionTracker[ipAddress].nEvictions - + (nRatioElapsed * mapInboundConnectionTracker[ipAddress].nEvictions); + if (nEvictions < 0) + nEvictions = 0; + } - // Identify the network group with the most connections and youngest member. - // (vEvictionCandidates is already sorted by reverse connect time) - uint64_t naMostConnections; - unsigned int nMostConnections = 0; - int64_t nMostConnectionsTime = 0; - std::map > mapNetGroupNodes; - for (const NodeEvictionCandidate &node : vEvictionCandidates) - { - mapNetGroupNodes[node.nKeyedNetGroup].push_back(node); - int64_t grouptime = mapNetGroupNodes[node.nKeyedNetGroup][0].nTimeConnected; - size_t groupsize = mapNetGroupNodes[node.nKeyedNetGroup].size(); + nEvictions += 1; + mapInboundConnectionTracker[ipAddress].nEvictions = nEvictions; + mapInboundConnectionTracker[ipAddress].nLastEvictionTime = GetTime(); - if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) + LogPrint("EVICT", "Number of Evictions is %f for %s\n", nEvictions, + vEvictionCandidatesByActivity[0]->addr.ToString()); + if (nEvictions > 15) { - nMostConnections = groupsize; - nMostConnectionsTime = grouptime; - naMostConnections = node.nKeyedNetGroup; + int nHoursToBan = 4; + Ban(ipAddress, BanReasonNodeMisbehaving, nHoursToBan * 60 * 60); + LogPrintf("Banning %s for %d hours: Too many evictions - connection dropped\n", + vEvictionCandidatesByActivity[0]->addr.ToString(), nHoursToBan); } } - // Reduce to the network group with the most connections - vEvictionCandidates = std::move(mapNetGroupNodes[naMostConnections]); - - // Disconnect from the network group with the most connections - NodeId evicted = vEvictionCandidates.front().id; - LOCK(cs_vNodes); - for (std::vector::const_iterator it(vNodes.begin()); it != vNodes.end(); ++it) + LogPrint("EVICT", "Node disconnected because too inactive:%d bytes of activity for peer %s\n", + vEvictionCandidatesByActivity[0]->nActivityBytes, vEvictionCandidatesByActivity[0]->addrName); + for (unsigned int i = 0; i < vEvictionCandidatesByActivity.size(); i++) { - if ((*it)->GetId() == evicted) - { - (*it)->fDisconnect = true; - return true; - } + LogPrint("EVICT", "Node %s bytes %d candidate %d\n", vEvictionCandidatesByActivity[i]->addrName, + vEvictionCandidatesByActivity[i]->nActivityBytes, i); } - return false; + + return true; } void CConnman::AcceptConnection(const ListenSocket &hListenSocket) @@ -1224,13 +1069,6 @@ void CConnman::AcceptConnection(const ListenSocket &hListenSocket) return; } - if (!fNetworkActive) - { - LogPrintf("connection from %s dropped: not accepting new connections\n", addr.ToString()); - CloseSocket(hSocket); - return; - } - if (!IsSelectableSocket(hSocket)) { LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); @@ -1266,6 +1104,50 @@ void CConnman::AcceptConnection(const ListenSocket &hListenSocket) } } + // If connection attempts exceeded within allowable timeframe then ban peer + { + double nConnections = 0; + LOCK(cs_mapInboundConnectionTracker); + int64_t now = GetTime(); + CNetAddr ipAddress = (CNetAddr)addr; + if (mapInboundConnectionTracker.count(ipAddress)) + { + // Decay the current number of connections (over 60 seconds) depending on the last connection attempt + int64_t nTimeElapsed = now - mapInboundConnectionTracker[ipAddress].nLastConnectionTime; + if (nTimeElapsed < 0) + nTimeElapsed = 0; + double nRatioElapsed = (double)nTimeElapsed / 60; + nConnections = mapInboundConnectionTracker[ipAddress].nConnections - + (nRatioElapsed * mapInboundConnectionTracker[ipAddress].nConnections); + if (nConnections < 0) + nConnections = 0; + } + else + { + ConnectionHistory ch; + ch.nConnections = 0.0; + ch.nLastConnectionTime = now; + ch.nEvictions = 0.0; + ch.nLastEvictionTime = now; + mapInboundConnectionTracker[ipAddress] = ch; + } + + nConnections += 1; + mapInboundConnectionTracker[ipAddress].nConnections = nConnections; + mapInboundConnectionTracker[ipAddress].nLastConnectionTime = GetTime(); + + LogPrint("EVICT", "Number of connection attempts is %f for %s\n", nConnections, addr.ToString()); + if (nConnections > 4 && !whitelisted && !addr.IsLocal()) // local connections are auto-whitelisted + { + int nHoursToBan = 4; + Ban((CNetAddr)addr, BanReasonNodeMisbehaving, nHoursToBan * 60 * 60); + LogPrintf("Banning %s for %d hours: Too many connection attempts - connection dropped\n", addr.ToString(), + nHoursToBan); + CloseSocket(hSocket); + return; + } + } + NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); @@ -1351,10 +1233,6 @@ void CConnman::ThreadSocketHandler() if (vNodesSize != nPrevNodeCount) { nPrevNodeCount = vNodesSize; - if (clientInterface) - { - clientInterface->NotifyNumConnectionsChanged(nPrevNodeCount); - } } // @@ -1538,7 +1416,6 @@ void CConnman::ThreadSocketHandler() pnode->nProcessQueueSize += nSizeAdded; pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; } - WakeMessageHandler(); } } else if (nBytes == 0) @@ -1623,15 +1500,6 @@ void CConnman::ThreadSocketHandler() } } -void CConnman::WakeMessageHandler() -{ - { - std::lock_guard lock(mutexMsgProc); - fMsgProcWake = true; - } - condMsgProc.notify_one(); -} - std::atomic upnp_thread_shutdown(false); #ifdef USE_UPNP @@ -2271,10 +2139,6 @@ bool CConnman::OpenNetworkConnection(const CAddress &addrConnect, { return false; } - if (!fNetworkActive) - { - return false; - } if (!pszDest) { if (IsLocal(addrConnect) || FindNode((CNetAddr)addrConnect) || IsBanned(addrConnect) || @@ -2322,7 +2186,7 @@ bool CConnman::OpenNetworkConnection(const CAddress &addrConnect, void CConnman::ThreadMessageHandler() { - while (!flagInterruptMsgProc) + while (interruptNet.load() == false) { std::vector vNodesCopy; { @@ -2344,21 +2208,13 @@ void CConnman::ThreadMessageHandler() } // Receive messages - bool fMoreNodeWork = GetNodeSignals().ProcessMessages(pnode, *this, flagInterruptMsgProc); + bool fMoreNodeWork = GetNodeSignals().ProcessMessages(pnode, *this); fMoreWork |= (fMoreNodeWork && !pnode->fPauseSend); - if (flagInterruptMsgProc) - { - return; - } // Send messages { LOCK(pnode->cs_sendProcessing); - GetNodeSignals().SendMessages(pnode, *this, flagInterruptMsgProc); - } - if (flagInterruptMsgProc) - { - return; + GetNodeSignals().SendMessages(pnode, *this); } } @@ -2370,13 +2226,10 @@ void CConnman::ThreadMessageHandler() } } - std::unique_lock lock(mutexMsgProc); if (!fMoreWork) { - condMsgProc.wait_until(lock, std::chrono::steady_clock::now() + std::chrono::milliseconds(100), - [this] { return fMsgProcWake; }); + MilliSleep(100); } - fMsgProcWake = false; } } @@ -2557,30 +2410,8 @@ void Discover(thread_group &threadGroup) #endif } -void CConnman::SetNetworkActive(bool active) +CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : nSeed0(nSeed0In), nSeed1(nSeed1In), netThreads(&interruptNet) { - LogPrintf("SetNetworkActive: %s\n", active); - - if (!active) - { - fNetworkActive = false; - - LOCK(cs_vNodes); - // Close sockets to all nodes - for (CNode *pnode : vNodes) - { - pnode->CloseSocketDisconnect(); - } - } - else - { - fNetworkActive = true; - } -} - -CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : nSeed0(nSeed0In), nSeed1(nSeed1In) -{ - fNetworkActive = true; setBannedIsDirty = false; fAddressesInitialized = false; nLastNodeId = 0; @@ -2592,39 +2423,49 @@ CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : nSeed0(nSeed0In), nSe nMaxOutbound = 0; nMaxAddnode = 0; nBestHeight = 0; - clientInterface = nullptr; - flagInterruptMsgProc = false; interruptNet.store(false); } NodeId CConnman::GetNewNodeId() { return nLastNodeId.fetch_add(1, std::memory_order_relaxed); } -bool CConnman::Start(std::string &strNodeError, Options connOptions) +ServiceFlags DEFAULT_RELEVANT_SERVICES = NODE_NETWORK; +ServiceFlags DEFAULT_LOCAL_SERVICES = NODE_NETWORK; + +extern int initMaxConnections; + +bool CConnman::Start(std::string &strNodeError) { + netThreads.clear(); nTotalBytesRecv = 0; nTotalBytesSent = 0; nMaxOutboundTotalBytesSentInCycle = 0; nMaxOutboundCycleStartTime = 0; - nRelevantServices = connOptions.nRelevantServices; - nLocalServices = connOptions.nLocalServices; - nMaxConnections = connOptions.nMaxConnections; - nMaxOutbound = std::min((connOptions.nMaxOutbound), nMaxConnections); - nMaxAddnode = connOptions.nMaxAddnode; - nMaxFeeler = connOptions.nMaxFeeler; + nRelevantServices = DEFAULT_RELEVANT_SERVICES; + nLocalServices = DEFAULT_LOCAL_SERVICES; + if (gArgs.GetBoolArg("-peerbloomfilters", true)) + { + nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); + } - nSendBufferMaxSize = connOptions.nSendBufferMaxSize; - nReceiveFloodSize = connOptions.nReceiveFloodSize; + nMaxConnections = initMaxConnections; + nMaxOutbound = std::min(MAX_OUTBOUND_CONNECTIONS, nMaxConnections); + nMaxAddnode = MAX_ADDNODE_CONNECTIONS; + nMaxFeeler = 1; - nMaxOutboundLimit = connOptions.nMaxOutboundLimit; - nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; + nSendBufferMaxSize = 1000 * gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); + nReceiveFloodSize = 1000 * gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); - SetBestHeight(connOptions.nBestHeight); + nMaxOutboundLimit = 0; - clientInterface = connOptions.uiInterface; - if (clientInterface) + if (gArgs.IsArgSet("-maxuploadtarget")) { - clientInterface->InitMessage(_("Loading addresses...")); + nMaxOutboundLimit = gArgs.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET) * 1024 * 1024; } + nMaxOutboundTimeframe = MAX_UPLOAD_TIMEFRAME; + + SetBestHeight(pnetMan->getChainActive()->chainActive.Height()); + + LogPrintf("Loading addresses..."); // Load addresses from peers.dat int64_t nStart = GetTimeMillis(); { @@ -2641,10 +2482,7 @@ bool CConnman::Start(std::string &strNodeError, Options connOptions) DumpAddresses(); } } - if (clientInterface) - { - clientInterface->InitMessage(_("Loading banlist...")); - } + LogPrintf("Loading banlist..."); // Load addresses from banlist.dat nStart = GetTimeMillis(); CBanDB bandb; @@ -2669,7 +2507,7 @@ bool CConnman::Start(std::string &strNodeError, Options connOptions) DumpBanlist(); } - uiInterface.InitMessage(_("Starting network threads...")); + LogPrintf("Starting network threads..."); fAddressesInitialized = true; @@ -2689,17 +2527,10 @@ bool CConnman::Start(std::string &strNodeError, Options connOptions) // Start threads // InterruptSocks5(false); - flagInterruptMsgProc = false; interruptNet.store(false); - { - std::unique_lock lock(mutexMsgProc); - fMsgProcWake = false; - } - // Send and receive from sockets, accept connections - threadSocketHandler = std::thread(&TraceThread >, "net", - std::function(std::bind(&CConnman::ThreadSocketHandler, this))); + netThreads.create_thread("net", &CConnman::ThreadSocketHandler, this); if (!gArgs.GetBoolArg("-dnsseed", true)) { @@ -2707,28 +2538,23 @@ bool CConnman::Start(std::string &strNodeError, Options connOptions) } else { - threadDNSAddressSeed = std::thread(&TraceThread >, "dnsseed", - std::function(std::bind(&CConnman::ThreadDNSAddressSeed, this))); + netThreads.create_thread("dnsseed", &CConnman::ThreadDNSAddressSeed, this); } // Initiate outbound connections from -addnode - threadOpenAddedConnections = std::thread(&TraceThread >, "addcon", - std::function(std::bind(&CConnman::ThreadOpenAddedConnections, this))); + netThreads.create_thread("addcon", &CConnman::ThreadOpenAddedConnections, this); // Initiate outbound connections unless connect=0 if (!gArgs.IsArgSet("-connect") || gArgs.GetArgs("-connect").size() != 1 || gArgs.GetArgs("-connect")[0] != "0") { - threadOpenConnections = std::thread(&TraceThread >, "opencon", - std::function(std::bind(&CConnman::ThreadOpenConnections, this))); + netThreads.create_thread("opencon", &CConnman::ThreadOpenConnections, this); } // Process messages - threadMessageHandler = std::thread(&TraceThread >, "msghand", - std::function(std::bind(&CConnman::ThreadMessageHandler, this))); + netThreads.create_thread("msghand", &CConnman::ThreadMessageHandler, this); // Dump network addresses - threadDumpData = std::thread(&TraceThread >, "dumpdata", - std::function(std::bind(&CConnman::DumpData, this, DUMP_ADDRESSES_INTERVAL))); + netThreads.create_thread("dumpdata", &CConnman::DumpData, this, DUMP_ADDRESSES_INTERVAL); return true; } @@ -2748,11 +2574,6 @@ class CNetCleanup void CConnman::Interrupt() { - { - std::lock_guard lock(mutexMsgProc); - flagInterruptMsgProc = true; - } - condMsgProc.notify_all(); interruptNet.store(true); InterruptSocks5(true); @@ -2775,30 +2596,8 @@ void CConnman::Interrupt() void CConnman::Stop() { - if (threadMessageHandler.joinable()) - { - threadMessageHandler.join(); - } - if (threadOpenConnections.joinable()) - { - threadOpenConnections.join(); - } - if (threadOpenAddedConnections.joinable()) - { - threadOpenAddedConnections.join(); - } - if (threadDNSAddressSeed.joinable()) - { - threadDNSAddressSeed.join(); - } - if (threadSocketHandler.joinable()) - { - threadSocketHandler.join(); - } - if (threadDumpData.joinable()) - { - threadDumpData.join(); - } + netThreads.interrupt_all(); + netThreads.join_all(); if (fAddressesInitialized) { @@ -3092,8 +2891,7 @@ CNode::CNode(NodeId idIn, bool fInboundIn) : nTimeConnected(GetSystemTimeInSeconds()), addr(addrIn), fInbound(fInboundIn), id(idIn), nKeyedNetGroup(nKeyedNetGroupIn), addrKnown(5000, 0.001), filterInventoryKnown(50000, 0.000001), - nLocalHostNonce(nLocalHostNonceIn), nLocalServices(nLocalServicesIn), nMyStartingHeight(nMyStartingHeightIn), - nSendVersion(0) + nLocalServices(nLocalServicesIn), nMyStartingHeight(nMyStartingHeightIn), nSendVersion(0) { nServices = NODE_NONE; nServicesExpected = NODE_NONE; @@ -3103,6 +2901,7 @@ CNode::CNode(NodeId idIn, nLastRecv = 0; nSendBytes = 0; nRecvBytes = 0; + nActivityBytes = 0; nTimeOffset = 0; addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; nVersion = 0; @@ -3221,51 +3020,6 @@ bool CConnman::NodeFullyConnected(const CNode *pnode) return pnode && pnode->fSuccessfullyConnected && !pnode->fDisconnect; } -void CConnman::PushMessage(CNode *pnode, CSerializedNetMsg &&msg) -{ - size_t nMessageSize = msg.data.size(); - size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrintf("sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->id); - - std::vector serializedHeader; - serializedHeader.reserve(CMessageHeader::HEADER_SIZE); - uint256 hash = Hash(msg.data.data(), msg.data.data() + nMessageSize); - CMessageHeader hdr(pnetMan->getActivePaymentNetwork()->MessageStart(), msg.command.c_str(), nMessageSize); - memcpy(hdr.pchChecksum, hash.begin(), CMessageHeader::CHECKSUM_SIZE); - - CVectorWriter{SER_NETWORK, MIN_PROTO_VERSION, serializedHeader, 0, hdr}; - - size_t nBytesSent = 0; - { - LOCK(pnode->cs_vSend); - bool optimisticSend(pnode->vSendMsg.empty()); - - // log total amount of bytes per command - pnode->mapSendBytesPerMsgCmd[msg.command] += nTotalSize; - pnode->nSendSize += nTotalSize; - - if (pnode->nSendSize > nSendBufferMaxSize) - { - pnode->fPauseSend = true; - } - pnode->vSendMsg.push_back(std::move(serializedHeader)); - if (nMessageSize) - { - pnode->vSendMsg.push_back(std::move(msg.data)); - } - - // If write queue empty, attempt "optimistic write" - if (optimisticSend == true) - { - nBytesSent = SocketSendData(pnode); - } - } - if (nBytesSent) - { - RecordBytesSent(nBytesSent); - } -} - bool CConnman::ForNode(NodeId id, std::function func) { CNode *found = nullptr; diff --git a/src/net.h b/src/net/net.h similarity index 84% rename from src/net.h rename to src/net/net.h index 2f98fcea..070e9dd3 100644 --- a/src/net.h +++ b/src/net/net.h @@ -30,14 +30,15 @@ #include #include "addrdb.h" -#include "addrman.h" #include "amount.h" #include "bloom.h" #include "compat.h" #include "crypto/hash.h" #include "limitedmap.h" -#include "netbase.h" -#include "protocol.h" +#include "net/addrman.h" +#include "net/netbase.h" +#include "net/protocol.h" +#include "networks/netman.h" #include "random.h" #include "streams.h" #include "sync.h" @@ -51,13 +52,6 @@ #include #include -class CNode; -class CScheduler; - -namespace boost -{ -class thread_group; -} // namespace boost /** Time between pings automatically sent out for latency probing and keepalive * (in seconds). */ @@ -117,7 +111,11 @@ static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; /** Subversion as sent to the P2P network in `version` messages */ extern std::string strSubVersion; +extern CNetworkManager *pnetMan; + typedef int64_t NodeId; +// Command, total bytes +typedef std::map mapMsgCmdSize; struct AddedNodeInfo { @@ -131,725 +129,786 @@ class CTransaction; class CNodeStats; class CClientUIInterface; -struct CSerializedNetMsg -{ - CSerializedNetMsg() = default; - CSerializedNetMsg(CSerializedNetMsg &&) = default; - CSerializedNetMsg &operator=(CSerializedNetMsg &&) = default; - // No copying, only moves. - CSerializedNetMsg(const CSerializedNetMsg &msg) = delete; - CSerializedNetMsg &operator=(const CSerializedNetMsg &) = delete; - - std::vector data; - std::string command; -}; -class CConnman +class CNetMessage { -public: - enum NumConnections - { - CONNECTIONS_NONE = 0, - CONNECTIONS_IN = (1U << 0), - CONNECTIONS_OUT = (1U << 1), - CONNECTIONS_ALL = (CONNECTIONS_IN | CONNECTIONS_OUT), - }; - - struct Options - { - ServiceFlags nLocalServices = NODE_NONE; - ServiceFlags nRelevantServices = NODE_NONE; - int nMaxConnections = 0; - int nMaxOutbound = 0; - int nMaxAddnode = 0; - int nMaxFeeler = 0; - int nBestHeight = 0; - CClientUIInterface *uiInterface = nullptr; - unsigned int nSendBufferMaxSize = 0; - unsigned int nReceiveFloodSize = 0; - uint64_t nMaxOutboundTimeframe = 0; - uint64_t nMaxOutboundLimit = 0; - }; - CConnman(uint64_t seed0, uint64_t seed1); - ~CConnman(); - bool Start(std::string &strNodeError, Options options); - void Stop(); - void Interrupt(); - bool BindListenPort(const CService &bindAddr, std::string &strError, bool fWhitelisted = false); - bool GetNetworkActive() const { return fNetworkActive; }; - void SetNetworkActive(bool active); - bool OpenNetworkConnection(const CAddress &addrConnect, - bool fCountFailure, - CSemaphoreGrant *grantOutbound = nullptr, - const char *strDest = nullptr, - bool fOneShot = false, - bool fFeeler = false, - bool fAddnode = false); - bool CheckIncomingNonce(uint64_t nonce); +private: + mutable CHash256 hasher; + mutable uint256 data_hash; - bool ForNode(NodeId id, std::function func); +public: + // Parsing header (false) or data (true) + bool in_data; - void PushMessage(CNode *pnode, CSerializedNetMsg &&msg); + // Partially received header. + CDataStream hdrbuf; + // Complete header. + CMessageHeader hdr; + unsigned int nHdrPos; - template - void ForEachNode(Callable &&func) - { - LOCK(cs_vNodes); - for (auto &&node : vNodes) - { - if (NodeFullyConnected(node)) - func(node); - } - }; + // Received message data. + CDataStream vRecv; + unsigned int nDataPos; - template - void ForEachNode(Callable &&func) const - { - LOCK(cs_vNodes); - for (auto &&node : vNodes) - { - if (NodeFullyConnected(node)) - func(node); - } - }; + // Time (in microseconds) of message receipt. + int64_t nTime; - template - void ForEachNodeThen(Callable &&pre, CallableAfter &&post) + CNetMessage(const CMessageHeader::MessageMagic &pchMessageStartIn, int nTypeIn, int nVersionIn) + : hdrbuf(nTypeIn, nVersionIn), hdr(pchMessageStartIn), vRecv(nTypeIn, nVersionIn) { - LOCK(cs_vNodes); - for (auto &&node : vNodes) - { - if (NodeFullyConnected(node)) - pre(node); - } - post(); - }; + hdrbuf.resize(24); + in_data = false; + nHdrPos = 0; + nDataPos = 0; + nTime = 0; + } - template - void ForEachNodeThen(Callable &&pre, CallableAfter &&post) const + bool complete() const { - LOCK(cs_vNodes); - for (auto &&node : vNodes) + if (!in_data) { - if (NodeFullyConnected(node)) - pre(node); + return false; } - post(); - }; - - // Addrman functions - size_t GetAddressCount() const; - void SetServices(const CService &addr, ServiceFlags nServices); - void MarkAddressGood(const CAddress &addr); - void AddNewAddress(const CAddress &addr, const CAddress &addrFrom, int64_t nTimePenalty = 0); - void AddNewAddresses(const std::vector &vAddr, const CAddress &addrFrom, int64_t nTimePenalty = 0); - std::vector GetAddresses(); - void AddressCurrentlyConnected(const CService &addr); - - // Denial-of-service detection/prevention. The idea is to detect peers that - // are behaving badly and disconnect/ban them, but do it in a - // one-coding-mistake-won't-shatter-the-entire-network way. - // IMPORTANT: There should be nothing I can give a node that it will forward - // on that will make that node's peers drop it. If there is, an attacker can - // isolate a node and/or try to split the network. Dropping a node for - // sending stuff that is invalid now but might be valid in a later version - // is also dangerous, because it can cause a network split between nodes - // running old code and nodes running new code. - void Ban(const CNetAddr &netAddr, const BanReason &reason, int64_t bantimeoffset = 0, bool sinceUnixEpoch = false); - void Ban(const CSubNet &subNet, const BanReason &reason, int64_t bantimeoffset = 0, bool sinceUnixEpoch = false); - // Needed for unit testing. - void ClearBanned(); - bool IsBanned(CNetAddr ip); - bool IsBanned(CSubNet subnet); - bool Unban(const CNetAddr &ip); - bool Unban(const CSubNet &ip); - void GetBanned(banmap_t &banmap); - void SetBanned(const banmap_t &banmap); - - void AddOneShot(const std::string &strDest); - bool AddNode(const std::string &node); - bool RemoveAddedNode(const std::string &node); - std::vector GetAddedNodeInfo(); + return (hdr.nMessageSize == nDataPos); + } - size_t GetNodeCount(NumConnections num); - void GetNodeStats(std::vector &vstats); - bool DisconnectNode(const std::string &node); - bool DisconnectNode(NodeId id); + const uint256 &GetMessageHash() const; - unsigned int GetSendBufferSize() const; + void SetVersion(int nVersionIn) + { + hdrbuf.SetVersion(nVersionIn); + vRecv.SetVersion(nVersionIn); + } - void AddWhitelistedRange(const CSubNet &subnet); + int readHeader(const char *pch, unsigned int nBytes); + int readData(const char *pch, unsigned int nBytes); +}; - ServiceFlags GetLocalServices() const; +/** Information about a peer */ +class CNode +{ + friend class CConnman; - //! set the max outbound target in bytes. - void SetMaxOutboundTarget(uint64_t limit); - uint64_t GetMaxOutboundTarget(); +public: + // socket + std::atomic nServices; + // Services expected from a peer, otherwise it will be disconnected + ServiceFlags nServicesExpected; + SOCKET hSocket; + // Total size of all vSendMsg entries. + size_t nSendSize; + // Offset inside the first vSendMsg already sent. + size_t nSendOffset; + uint64_t nSendBytes; + // Total bytes sent and received + uint64_t nActivityBytes; + std::deque > vSendMsg; + CCriticalSection cs_vSend; + CCriticalSection cs_hSocket; + CCriticalSection cs_vRecv; - //! set the timeframe for the max outbound target. - void SetMaxOutboundTimeframe(uint64_t timeframe); - uint64_t GetMaxOutboundTimeframe(); + CCriticalSection cs_vProcessMsg; + std::list vProcessMsg; + size_t nProcessQueueSize; - //! check if the outbound target is reached. - // If param historicalBlockServingLimit is set true, the function will - // response true if the limit for serving historical blocks has been - // reached. - bool OutboundTargetReached(bool historicalBlockServingLimit); + CCriticalSection cs_sendProcessing; - //! response the bytes left in the current max outbound cycle - // in case of no limit, it will always response 0 - uint64_t GetOutboundTargetBytesLeft(); + std::deque vRecvGetData; + uint64_t nRecvBytes; + std::atomic nRecvVersion; - //! response the time in second left in the current max outbound cycle - // in case of no limit, it will always response 0 - uint64_t GetMaxOutboundTimeLeftInCycle(); + std::atomic nLastSend; + std::atomic nLastRecv; + const int64_t nTimeConnected; + std::atomic nTimeOffset; + const CAddress addr; + std::atomic nVersion; + // strSubVer is whatever byte array we read from the wire. However, this + // field is intended to be printed out, displayed to humans in various forms + // and so on. So we sanitize it and store the sanitized version in + // cleanSubVer. The original should be used when dealing with the network or + // wire types and the cleaned string used when displayed or logged. + std::string strSubVer, cleanSubVer; + // Used for both cleanSubVer and strSubVer. + CCriticalSection cs_SubVer; + // This peer can bypass DoS banning. + bool fWhitelisted; + // If true this node is being used as a short lived feeler. + bool fFeeler; + bool fOneShot; + bool fAddnode; + bool fClient; + const bool fInbound; + std::atomic_bool fSuccessfullyConnected; + std::atomic_bool fDisconnect; + // We use fRelayTxes for two purposes - + // a) it allows us to not relay tx invs before receiving the peer's version + // message. + // b) the peer may tell us in its version message that we should not relay + // tx invs unless it loads a bloom filter. - uint64_t GetTotalBytesRecv(); - uint64_t GetTotalBytesSent(); + // protected by cs_filter + bool fRelayTxes; + bool fSentAddr; + CSemaphoreGrant grantOutbound; + CCriticalSection cs_filter; + CBloomFilter *pfilter; + std::atomic nRefCount; + const NodeId id; - void SetBestHeight(int height); - int GetBestHeight() const; + const uint64_t nKeyedNetGroup; + std::atomic_bool fPauseRecv; + std::atomic_bool fPauseSend; - /** Get a unique deterministic randomizer. */ - CSipHasher GetDeterministicRandomizer(uint64_t id) const; +protected: + mapMsgCmdSize mapSendBytesPerMsgCmd; + mapMsgCmdSize mapRecvBytesPerMsgCmd; - unsigned int GetReceiveFloodSize() const; +public: + uint256 hashContinue; + std::atomic nStartingHeight; - void WakeMessageHandler(); + // flood relay + std::vector vAddrToSend; + CRollingBloomFilter addrKnown; + bool fGetAddr; + std::set setKnown; + int64_t nNextAddrSend; + int64_t nNextLocalAddrSend; -private: - struct ListenSocket - { - SOCKET socket; - bool whitelisted; - - ListenSocket(SOCKET socket_, bool whitelisted_) : socket(socket_), whitelisted(whitelisted_) {} - }; - - void ThreadOpenAddedConnections(); - void ProcessOneShot(); - void ThreadOpenConnections(); - void ThreadMessageHandler(); - void AcceptConnection(const ListenSocket &hListenSocket); - void ThreadSocketHandler(); - void ThreadDNSAddressSeed(); + // Inventory based relay. + CRollingBloomFilter filterInventoryKnown; + // Set of transaction ids we still have to announce. They are sorted by the + // mempool before relay, so the order is not important. + std::set setInventoryTxToSend; + // List of block ids we still have announce. There is no final sorting + // before sending, as they are always sent immediately and in the order + // requested. + std::vector vInventoryBlockToSend; + CCriticalSection cs_inventory; + std::set setAskFor; + std::multimap mapAskFor; + int64_t nNextInvSend; + // Used for headers announcements - unfiltered blocks to relay. Also + // protected by cs_inventory. + std::vector vBlockHashesToAnnounce; + // Used for BIP35 mempool sending, also protected by cs_inventory. + bool fSendMempool; - uint64_t CalculateKeyedNetGroup(const CAddress &ad) const; + // Last time a "MEMPOOL" request was serviced. + std::atomic timeLastMempoolReq; - CNode *FindNode(const CNetAddr &ip); - CNode *FindNode(const CSubNet &subNet); - CNode *FindNode(const std::string &addrName); - CNode *FindNode(const CService &addr); + // Block and TXN accept times + std::atomic nLastBlockTime; + std::atomic nLastTXTime; - bool AttemptToEvictConnection(); - CNode *ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure); - bool IsWhitelistedRange(const CNetAddr &addr); + // Ping time measurement: + // The pong reply we're expecting, or 0 if no pong expected. + std::atomic nPingNonceSent; + // Time (in usec) the last ping was sent, or 0 if no ping was ever sent. + std::atomic nPingUsecStart; + // Last measured round-trip time. + std::atomic nPingUsecTime; + // Best measured round-trip time. + std::atomic nMinPingUsecTime; + // Whether a ping is requested. + std::atomic fPingQueued; - void DeleteNode(CNode *pnode); + CNode(NodeId id, + ServiceFlags nLocalServicesIn, + int nMyStartingHeightIn, + SOCKET hSocketIn, + const CAddress &addrIn, + uint64_t nKeyedNetGroupIn, + uint64_t nLocalHostNonceIn, + const std::string &addrNameIn = "", + bool fInboundIn = false); + ~CNode(); - NodeId GetNewNodeId(); +private: + CNode(const CNode &); + void operator=(const CNode &); - size_t SocketSendData(CNode *pnode) const; - //! check is the banlist has unwritten changes - bool BannedSetIsDirty(); - //! set the "dirty" flag for the banlist - void SetBannedSetDirty(bool dirty = true); - //! clean unused entries (if bantime has expired) - void SweepBanned(); - void DumpAddresses(); - void _DumpData(); - void DumpData(int64_t seconds_between_runs); - void DumpBanlist(); + // Services offered to this peer + const ServiceFlags nLocalServices; + const int nMyStartingHeight; + int nSendVersion; + // Used only by SocketHandler thread. + std::list vRecvMsg; - // Network stats - void RecordBytesRecv(uint64_t bytes); - void RecordBytesSent(uint64_t bytes); + mutable CCriticalSection cs_addrName; + std::string addrName; - // Whether the node should be passed out in ForEach* callbacks - static bool NodeFullyConnected(const CNode *pnode); + CService addrLocal; + mutable CCriticalSection cs_addrLocal; - // Network usage totals - CCriticalSection cs_totalBytesRecv; - CCriticalSection cs_totalBytesSent; - uint64_t nTotalBytesRecv; - uint64_t nTotalBytesSent; +public: + NodeId GetId() const { return id; } + int GetMyStartingHeight() const { return nMyStartingHeight; } + int GetRefCount() + { + assert(nRefCount >= 0); + return nRefCount; + } - // outbound limit & stats - uint64_t nMaxOutboundTotalBytesSentInCycle; - uint64_t nMaxOutboundCycleStartTime; - uint64_t nMaxOutboundLimit; - uint64_t nMaxOutboundTimeframe; + bool ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool &complete); - // Whitelisted ranges. Any node connecting from these is automatically - // whitelisted (as well as those connecting to whitelisted binds). - std::vector vWhitelistedRange; - CCriticalSection cs_vWhitelistedRange; + void SetRecvVersion(int nVersionIn) { nRecvVersion = nVersionIn; } + int GetRecvVersion() { return nRecvVersion; } + void SetSendVersion(int nVersionIn); + int GetSendVersion() const; - unsigned int nSendBufferMaxSize; - unsigned int nReceiveFloodSize; + CService GetAddrLocal() const; + //! May not be called more than once + void SetAddrLocal(const CService &addrLocalIn); - std::vector vhListenSocket; - std::atomic fNetworkActive; - banmap_t setBanned; - CCriticalSection cs_setBanned; - bool setBannedIsDirty; - bool fAddressesInitialized; - CAddrMan addrman; - std::deque vOneShots; - CCriticalSection cs_vOneShots; - std::vector vAddedNodes; - CCriticalSection cs_vAddedNodes; - std::vector vNodes; - std::list vNodesDisconnected; - mutable CCriticalSection cs_vNodes; - std::atomic nLastNodeId; + CNode *AddRef() + { + nRefCount++; + return this; + } - /** Services this instance offers */ - ServiceFlags nLocalServices; + void Release() { nRefCount--; } + void AddAddressKnown(const CAddress &_addr) { addrKnown.insert(_addr.GetKey()); } + void PushAddress(const CAddress &_addr, FastRandomContext &insecure_rand) + { + // Known checking here is only to save space from duplicates. + // SendMessages will filter it again for knowns that were added + // after addresses were pushed. + if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) + { + if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) + { + vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; + } + else + { + vAddrToSend.push_back(_addr); + } + } + } - /** Services this instance cares about */ - ServiceFlags nRelevantServices; + void AddInventoryKnown(const CInv &inv) + { + LOCK(cs_inventory); + filterInventoryKnown.insert(inv.hash); + } - std::unique_ptr semOutbound; - std::unique_ptr semAddnode; - int nMaxConnections; - int nMaxOutbound; - int nMaxAddnode; - int nMaxFeeler; - std::atomic nBestHeight; - CClientUIInterface *clientInterface; + void PushInventory(const CInv &inv) + { + LOCK(cs_inventory); + if (inv.type == MSG_TX) + { + if (!filterInventoryKnown.contains(inv.hash)) + { + setInventoryTxToSend.insert(inv.hash); + } + } + else if (inv.type == MSG_BLOCK) + { + vInventoryBlockToSend.push_back(inv.hash); + } + } - /** SipHasher seeds for deterministic randomness */ - const uint64_t nSeed0, nSeed1; + void PushBlockHash(const uint256 &hash) + { + LOCK(cs_inventory); + vBlockHashesToAnnounce.push_back(hash); + } - /** flag for waking the message processor. */ - bool fMsgProcWake; + void AskFor(const CInv &inv); - std::condition_variable condMsgProc; - std::mutex mutexMsgProc; - std::atomic flagInterruptMsgProc; + void CloseSocketDisconnect(); - std::atomic interruptNet; + void copyStats(CNodeStats &stats); - std::thread threadDNSAddressSeed; - std::thread threadSocketHandler; - std::thread threadOpenAddedConnections; - std::thread threadOpenConnections; - std::thread threadMessageHandler; - std::thread threadDumpData; + ServiceFlags GetLocalServices() const { return nLocalServices; } + std::string GetAddrName() const; + //! Sets the addrName only if it was not previously set + void MaybeSetAddrName(const std::string &addrNameIn); }; -extern std::unique_ptr g_connman; -void Discover(thread_group &threadGroup); -void MapPort(bool fUseUPnP); -unsigned short GetListenPort(); -bool BindListenPort(const CService &bindAddr, std::string &strError, bool fWhitelisted = false); - -struct CombinerAll +class CConnman { - typedef bool result_type; +public: + enum NumConnections + { + CONNECTIONS_NONE = 0, + CONNECTIONS_IN = (1U << 0), + CONNECTIONS_OUT = (1U << 1), + CONNECTIONS_ALL = (CONNECTIONS_IN | CONNECTIONS_OUT), + }; - template - bool operator()(I first, I last) const + CConnman(uint64_t seed0, uint64_t seed1); + ~CConnman(); + bool Start(std::string &strNodeError); + void Stop(); + void Interrupt(); + bool BindListenPort(const CService &bindAddr, std::string &strError, bool fWhitelisted = false); + bool OpenNetworkConnection(const CAddress &addrConnect, + bool fCountFailure, + CSemaphoreGrant *grantOutbound = nullptr, + const char *strDest = nullptr, + bool fOneShot = false, + bool fFeeler = false, + bool fAddnode = false); + + bool ForNode(NodeId id, std::function func); + + template + void PushMessage(CNode *pnode, std::string sCommand, Args &&... args) { - while (first != last) + std::vector data; + CVectorWriter{SER_NETWORK, pnode->GetSendVersion(), data, 0, std::forward(args)...}; + size_t nMessageSize = data.size(); + size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; + LogPrintf("sending %s (%d bytes) peer=%d\n", SanitizeString(sCommand.c_str()), nMessageSize, pnode->id); + + std::vector serializedHeader; + serializedHeader.reserve(CMessageHeader::HEADER_SIZE); + uint256 hash = Hash(data.data(), data.data() + nMessageSize); + CMessageHeader hdr(pnetMan->getActivePaymentNetwork()->MessageStart(), sCommand.c_str(), nMessageSize); + memcpy(hdr.pchChecksum, hash.begin(), CMessageHeader::CHECKSUM_SIZE); + + CVectorWriter{SER_NETWORK, MIN_PROTO_VERSION, serializedHeader, 0, hdr}; + + size_t nBytesSent = 0; { - if (!(*first)) + LOCK(pnode->cs_vSend); + bool optimisticSend(pnode->vSendMsg.empty()); + + // log total amount of bytes per command + pnode->mapSendBytesPerMsgCmd[sCommand] += nTotalSize; + pnode->nSendSize += nTotalSize; + + if (pnode->nSendSize > nSendBufferMaxSize) { - return false; + pnode->fPauseSend = true; + } + pnode->vSendMsg.push_back(std::move(serializedHeader)); + if (nMessageSize) + { + pnode->vSendMsg.push_back(std::move(data)); + } + const char *strCommand = sCommand.c_str(); + if (strcmp(strCommand, NetMsgType::PING) != 0 && strcmp(strCommand, NetMsgType::PONG) != 0 && + strcmp(strCommand, NetMsgType::ADDR) != 0 && strcmp(strCommand, NetMsgType::VERSION) != 0 && + strcmp(strCommand, NetMsgType::VERACK) != 0 && strcmp(strCommand, NetMsgType::INV) != 0) + { + pnode->nActivityBytes += nMessageSize; } - ++first; - } - return true; - } -}; - -// Signals for message handling -struct CNodeSignals -{ - boost::signals2::signal &), CombinerAll> ProcessMessages; - boost::signals2::signal &), CombinerAll> SendMessages; - boost::signals2::signal InitializeNode; - boost::signals2::signal FinalizeNode; -}; - -CNodeSignals &GetNodeSignals(); -enum -{ - // unknown - LOCAL_NONE, - // address a local interface listens on - LOCAL_IF, - // address explicit bound to - LOCAL_BIND, - // address reported by UPnP - LOCAL_UPNP, - // address explicitly specified (-externalip=) - LOCAL_MANUAL, + // If write queue empty, attempt "optimistic write" + if (optimisticSend == true) + { + nBytesSent = SocketSendData(pnode); + } + } + if (nBytesSent) + { + RecordBytesSent(nBytesSent); + } + } - LOCAL_MAX -}; + template + void ForEachNode(Callable &&func) + { + LOCK(cs_vNodes); + for (auto &&node : vNodes) + { + if (NodeFullyConnected(node)) + func(node); + } + }; -bool IsPeerAddrLocalGood(CNode *pnode); -void AdvertiseLocal(CNode *pnode); -void SetLimited(enum Network net, bool fLimited = true); -bool IsLimited(enum Network net); -bool IsLimited(const CNetAddr &addr); -bool AddLocal(const CService &addr, int nScore = LOCAL_NONE); -bool AddLocal(const CNetAddr &addr, int nScore = LOCAL_NONE); -bool RemoveLocal(const CService &addr); -bool SeenLocal(const CService &addr); -bool IsLocal(const CService &addr); -bool GetLocal(CService &addr, const CNetAddr *paddrPeer = nullptr); -bool IsReachable(enum Network net); -bool IsReachable(const CNetAddr &addr); -CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices); + template + void ForEachNode(Callable &&func) const + { + LOCK(cs_vNodes); + for (auto &&node : vNodes) + { + if (NodeFullyConnected(node)) + func(node); + } + }; -extern bool fDiscover; -extern bool fListen; -extern bool fRelayTxes; + template + void ForEachNodeThen(Callable &&pre, CallableAfter &&post) + { + LOCK(cs_vNodes); + for (auto &&node : vNodes) + { + if (NodeFullyConnected(node)) + pre(node); + } + post(); + }; -extern limitedmap mapAlreadyAskedFor; + template + void ForEachNodeThen(Callable &&pre, CallableAfter &&post) const + { + LOCK(cs_vNodes); + for (auto &&node : vNodes) + { + if (NodeFullyConnected(node)) + pre(node); + } + post(); + }; -struct LocalServiceInfo -{ - int nScore; - int nPort; -}; + // Addrman functions + size_t GetAddressCount() const; + void SetServices(const CService &addr, ServiceFlags nServices); + void MarkAddressGood(const CAddress &addr); + void AddNewAddress(const CAddress &addr, const CAddress &addrFrom, int64_t nTimePenalty = 0); + void AddNewAddresses(const std::vector &vAddr, const CAddress &addrFrom, int64_t nTimePenalty = 0); + std::vector GetAddresses(); + void AddressCurrentlyConnected(const CService &addr); -extern CCriticalSection cs_mapLocalHost; -extern std::map mapLocalHost; + // Denial-of-service detection/prevention. The idea is to detect peers that + // are behaving badly and disconnect/ban them, but do it in a + // one-coding-mistake-won't-shatter-the-entire-network way. + // IMPORTANT: There should be nothing I can give a node that it will forward + // on that will make that node's peers drop it. If there is, an attacker can + // isolate a node and/or try to split the network. Dropping a node for + // sending stuff that is invalid now but might be valid in a later version + // is also dangerous, because it can cause a network split between nodes + // running old code and nodes running new code. + void Ban(const CNetAddr &netAddr, const BanReason &reason, int64_t bantimeoffset = 0, bool sinceUnixEpoch = false); + void Ban(const CSubNet &subNet, const BanReason &reason, int64_t bantimeoffset = 0, bool sinceUnixEpoch = false); + // Needed for unit testing. + void ClearBanned(); + bool IsBanned(CNetAddr ip); + bool IsBanned(CSubNet subnet); + bool Unban(const CNetAddr &ip); + bool Unban(const CSubNet &ip); + void GetBanned(banmap_t &banmap); + void SetBanned(const banmap_t &banmap); -// Command, total bytes -typedef std::map mapMsgCmdSize; + void AddOneShot(const std::string &strDest); -class CNodeStats -{ -public: - NodeId nodeid; - ServiceFlags nServices; - bool fRelayTxes; - int64_t nLastSend; - int64_t nLastRecv; - int64_t nTimeConnected; - int64_t nTimeOffset; - std::string addrName; - int nVersion; - std::string cleanSubVer; - bool fInbound; - bool fAddnode; - int nStartingHeight; - uint64_t nSendBytes; - mapMsgCmdSize mapSendBytesPerMsgCmd; - uint64_t nRecvBytes; - mapMsgCmdSize mapRecvBytesPerMsgCmd; - bool fWhitelisted; - double dPingTime; - double dPingWait; - double dMinPing; - std::string addrLocal; - CAddress addr; -}; + bool AddNode(const std::string &node); + bool RemoveAddedNode(const std::string &node); + std::vector GetAddedNodeInfo(); -class CNetMessage -{ -private: - mutable CHash256 hasher; - mutable uint256 data_hash; + size_t GetNodeCount(NumConnections num); + void GetNodeStats(std::vector &vstats); + bool DisconnectNode(const std::string &node); + bool DisconnectNode(NodeId id); -public: - // Parsing header (false) or data (true) - bool in_data; + unsigned int GetSendBufferSize() const; - // Partially received header. - CDataStream hdrbuf; - // Complete header. - CMessageHeader hdr; - unsigned int nHdrPos; + void AddWhitelistedRange(const CSubNet &subnet); - // Received message data. - CDataStream vRecv; - unsigned int nDataPos; + ServiceFlags GetLocalServices() const; - // Time (in microseconds) of message receipt. - int64_t nTime; + //! set the max outbound target in bytes. + void SetMaxOutboundTarget(uint64_t limit); + uint64_t GetMaxOutboundTarget(); - CNetMessage(const CMessageHeader::MessageMagic &pchMessageStartIn, int nTypeIn, int nVersionIn) - : hdrbuf(nTypeIn, nVersionIn), hdr(pchMessageStartIn), vRecv(nTypeIn, nVersionIn) - { - hdrbuf.resize(24); - in_data = false; - nHdrPos = 0; - nDataPos = 0; - nTime = 0; - } + //! set the timeframe for the max outbound target. + void SetMaxOutboundTimeframe(uint64_t timeframe); + uint64_t GetMaxOutboundTimeframe(); - bool complete() const - { - if (!in_data) - { - return false; - } + //! check if the outbound target is reached. + // If param historicalBlockServingLimit is set true, the function will + // response true if the limit for serving historical blocks has been + // reached. + bool OutboundTargetReached(bool historicalBlockServingLimit); - return (hdr.nMessageSize == nDataPos); - } + //! response the bytes left in the current max outbound cycle + // in case of no limit, it will always response 0 + uint64_t GetOutboundTargetBytesLeft(); - const uint256 &GetMessageHash() const; + //! response the time in second left in the current max outbound cycle + // in case of no limit, it will always response 0 + uint64_t GetMaxOutboundTimeLeftInCycle(); - void SetVersion(int nVersionIn) - { - hdrbuf.SetVersion(nVersionIn); - vRecv.SetVersion(nVersionIn); - } + uint64_t GetTotalBytesRecv(); + uint64_t GetTotalBytesSent(); - int readHeader(const char *pch, unsigned int nBytes); - int readData(const char *pch, unsigned int nBytes); -}; + void SetBestHeight(int height); + int GetBestHeight() const; -/** Information about a peer */ -class CNode -{ - friend class CConnman; + /** Get a unique deterministic randomizer. */ + CSipHasher GetDeterministicRandomizer(uint64_t id) const; -public: - // socket - std::atomic nServices; - // Services expected from a peer, otherwise it will be disconnected - ServiceFlags nServicesExpected; - SOCKET hSocket; - // Total size of all vSendMsg entries. - size_t nSendSize; - // Offset inside the first vSendMsg already sent. - size_t nSendOffset; - uint64_t nSendBytes; - std::deque > vSendMsg; - CCriticalSection cs_vSend; - CCriticalSection cs_hSocket; - CCriticalSection cs_vRecv; + unsigned int GetReceiveFloodSize() const; - CCriticalSection cs_vProcessMsg; - std::list vProcessMsg; - size_t nProcessQueueSize; +private: + struct ListenSocket + { + SOCKET socket; + bool whitelisted; - CCriticalSection cs_sendProcessing; + ListenSocket(SOCKET socket_, bool whitelisted_) : socket(socket_), whitelisted(whitelisted_) {} + }; - std::deque vRecvGetData; - uint64_t nRecvBytes; - std::atomic nRecvVersion; + void ThreadOpenAddedConnections(); + void ProcessOneShot(); + void ThreadOpenConnections(); + void ThreadMessageHandler(); + void AcceptConnection(const ListenSocket &hListenSocket); + void ThreadSocketHandler(); + void ThreadDNSAddressSeed(); - std::atomic nLastSend; - std::atomic nLastRecv; - const int64_t nTimeConnected; - std::atomic nTimeOffset; - const CAddress addr; - std::atomic nVersion; - // strSubVer is whatever byte array we read from the wire. However, this - // field is intended to be printed out, displayed to humans in various forms - // and so on. So we sanitize it and store the sanitized version in - // cleanSubVer. The original should be used when dealing with the network or - // wire types and the cleaned string used when displayed or logged. - std::string strSubVer, cleanSubVer; - // Used for both cleanSubVer and strSubVer. - CCriticalSection cs_SubVer; - // This peer can bypass DoS banning. - bool fWhitelisted; - // If true this node is being used as a short lived feeler. - bool fFeeler; - bool fOneShot; - bool fAddnode; - bool fClient; - const bool fInbound; - std::atomic_bool fSuccessfullyConnected; - std::atomic_bool fDisconnect; - // We use fRelayTxes for two purposes - - // a) it allows us to not relay tx invs before receiving the peer's version - // message. - // b) the peer may tell us in its version message that we should not relay - // tx invs unless it loads a bloom filter. + uint64_t CalculateKeyedNetGroup(const CAddress &ad) const; - // protected by cs_filter - bool fRelayTxes; - bool fSentAddr; - CSemaphoreGrant grantOutbound; - CCriticalSection cs_filter; - CBloomFilter *pfilter; - std::atomic nRefCount; - const NodeId id; + CNode *FindNode(const CNetAddr &ip); + CNode *FindNode(const CSubNet &subNet); + CNode *FindNode(const std::string &addrName); + CNode *FindNode(const CService &addr); - const uint64_t nKeyedNetGroup; - std::atomic_bool fPauseRecv; - std::atomic_bool fPauseSend; + bool AttemptToEvictConnection(); + CNode *ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure); + bool IsWhitelistedRange(const CNetAddr &addr); -protected: - mapMsgCmdSize mapSendBytesPerMsgCmd; - mapMsgCmdSize mapRecvBytesPerMsgCmd; + void DeleteNode(CNode *pnode); -public: - uint256 hashContinue; - std::atomic nStartingHeight; + NodeId GetNewNodeId(); - // flood relay - std::vector vAddrToSend; - CRollingBloomFilter addrKnown; - bool fGetAddr; - std::set setKnown; - int64_t nNextAddrSend; - int64_t nNextLocalAddrSend; + size_t SocketSendData(CNode *pnode) const; + //! check is the banlist has unwritten changes + bool BannedSetIsDirty(); + //! set the "dirty" flag for the banlist + void SetBannedSetDirty(bool dirty = true); + //! clean unused entries (if bantime has expired) + void SweepBanned(); + void DumpAddresses(); + void _DumpData(); + void DumpData(int64_t seconds_between_runs); + void DumpBanlist(); - // Inventory based relay. - CRollingBloomFilter filterInventoryKnown; - // Set of transaction ids we still have to announce. They are sorted by the - // mempool before relay, so the order is not important. - std::set setInventoryTxToSend; - // List of block ids we still have announce. There is no final sorting - // before sending, as they are always sent immediately and in the order - // requested. - std::vector vInventoryBlockToSend; - CCriticalSection cs_inventory; - std::set setAskFor; - std::multimap mapAskFor; - int64_t nNextInvSend; - // Used for headers announcements - unfiltered blocks to relay. Also - // protected by cs_inventory. - std::vector vBlockHashesToAnnounce; - // Used for BIP35 mempool sending, also protected by cs_inventory. - bool fSendMempool; + // Network stats + void RecordBytesRecv(uint64_t bytes); + void RecordBytesSent(uint64_t bytes); - // Last time a "MEMPOOL" request was serviced. - std::atomic timeLastMempoolReq; + // Whether the node should be passed out in ForEach* callbacks + static bool NodeFullyConnected(const CNode *pnode); - // Block and TXN accept times - std::atomic nLastBlockTime; - std::atomic nLastTXTime; + // Network usage totals + CCriticalSection cs_totalBytesRecv; + CCriticalSection cs_totalBytesSent; + uint64_t nTotalBytesRecv; + uint64_t nTotalBytesSent; - // Ping time measurement: - // The pong reply we're expecting, or 0 if no pong expected. - std::atomic nPingNonceSent; - // Time (in usec) the last ping was sent, or 0 if no ping was ever sent. - std::atomic nPingUsecStart; - // Last measured round-trip time. - std::atomic nPingUsecTime; - // Best measured round-trip time. - std::atomic nMinPingUsecTime; - // Whether a ping is requested. - std::atomic fPingQueued; + // outbound limit & stats + uint64_t nMaxOutboundTotalBytesSentInCycle; + uint64_t nMaxOutboundCycleStartTime; + uint64_t nMaxOutboundLimit; + uint64_t nMaxOutboundTimeframe; - CNode(NodeId id, - ServiceFlags nLocalServicesIn, - int nMyStartingHeightIn, - SOCKET hSocketIn, - const CAddress &addrIn, - uint64_t nKeyedNetGroupIn, - uint64_t nLocalHostNonceIn, - const std::string &addrNameIn = "", - bool fInboundIn = false); - ~CNode(); + // Whitelisted ranges. Any node connecting from these is automatically + // whitelisted (as well as those connecting to whitelisted binds). + std::vector vWhitelistedRange; + CCriticalSection cs_vWhitelistedRange; -private: - CNode(const CNode &); - void operator=(const CNode &); + unsigned int nSendBufferMaxSize; + unsigned int nReceiveFloodSize; - const uint64_t nLocalHostNonce; - // Services offered to this peer - const ServiceFlags nLocalServices; - const int nMyStartingHeight; - int nSendVersion; - // Used only by SocketHandler thread. - std::list vRecvMsg; + std::vector vhListenSocket; + banmap_t setBanned; + CCriticalSection cs_setBanned; + bool setBannedIsDirty; + bool fAddressesInitialized; + CAddrMan addrman; + std::deque vOneShots; + CCriticalSection cs_vOneShots; + std::vector vAddedNodes; + CCriticalSection cs_vAddedNodes; + std::vector vNodes; + std::list vNodesDisconnected; + mutable CCriticalSection cs_vNodes; + std::atomic nLastNodeId; - mutable CCriticalSection cs_addrName; - std::string addrName; + /** Services this instance offers */ + ServiceFlags nLocalServices; - CService addrLocal; - mutable CCriticalSection cs_addrLocal; + /** Services this instance cares about */ + ServiceFlags nRelevantServices; -public: - NodeId GetId() const { return id; } - uint64_t GetLocalNonce() const { return nLocalHostNonce; } - int GetMyStartingHeight() const { return nMyStartingHeight; } - int GetRefCount() - { - assert(nRefCount >= 0); - return nRefCount; - } + std::unique_ptr semOutbound; + std::unique_ptr semAddnode; + int nMaxConnections; + int nMaxOutbound; + int nMaxAddnode; + int nMaxFeeler; + std::atomic nBestHeight; - bool ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool &complete); + /** SipHasher seeds for deterministic randomness */ + const uint64_t nSeed0, nSeed1; - void SetRecvVersion(int nVersionIn) { nRecvVersion = nVersionIn; } - int GetRecvVersion() { return nRecvVersion; } - void SetSendVersion(int nVersionIn); - int GetSendVersion() const; + std::atomic interruptNet; + thread_group netThreads; +}; - CService GetAddrLocal() const; - //! May not be called more than once - void SetAddrLocal(const CService &addrLocalIn); +extern std::unique_ptr g_connman; - CNode *AddRef() - { - nRefCount++; - return this; - } +void Discover(thread_group &threadGroup); +void MapPort(bool fUseUPnP); +unsigned short GetListenPort(); +bool BindListenPort(const CService &bindAddr, std::string &strError, bool fWhitelisted = false); - void Release() { nRefCount--; } - void AddAddressKnown(const CAddress &_addr) { addrKnown.insert(_addr.GetKey()); } - void PushAddress(const CAddress &_addr, FastRandomContext &insecure_rand) +struct CombinerAll +{ + typedef bool result_type; + + template + bool operator()(I first, I last) const { - // Known checking here is only to save space from duplicates. - // SendMessages will filter it again for knowns that were added - // after addresses were pushed. - if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) + while (first != last) { - if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) - { - vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; - } - else + if (!(*first)) { - vAddrToSend.push_back(_addr); + return false; } + ++first; } + return true; } +}; - void AddInventoryKnown(const CInv &inv) +// Signals for message handling +struct CNodeSignals +{ + boost::signals2::signal ProcessMessages; + boost::signals2::signal SendMessages; + boost::signals2::signal InitializeNode; + boost::signals2::signal FinalizeNode; +}; + +CNodeSignals &GetNodeSignals(); + +enum +{ + // unknown + LOCAL_NONE, + // address a local interface listens on + LOCAL_IF, + // address explicit bound to + LOCAL_BIND, + // address reported by UPnP + LOCAL_UPNP, + // address explicitly specified (-externalip=) + LOCAL_MANUAL, + + LOCAL_MAX +}; + +bool IsPeerAddrLocalGood(CNode *pnode); +void AdvertiseLocal(CNode *pnode); +void SetLimited(enum Network net, bool fLimited = true); +bool IsLimited(enum Network net); +bool IsLimited(const CNetAddr &addr); +bool AddLocal(const CService &addr, int nScore = LOCAL_NONE); +bool AddLocal(const CNetAddr &addr, int nScore = LOCAL_NONE); +bool RemoveLocal(const CService &addr); +bool SeenLocal(const CService &addr); +bool IsLocal(const CService &addr); +bool GetLocal(CService &addr, const CNetAddr *paddrPeer = nullptr); +bool IsReachable(enum Network net); +bool IsReachable(const CNetAddr &addr); +CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices); + +extern bool fDiscover; +extern bool fListen; +extern bool fRelayTxes; + +extern limitedmap mapAlreadyAskedFor; + +struct LocalServiceInfo +{ + int nScore; + int nPort; +}; + +extern CCriticalSection cs_mapLocalHost; +extern std::map mapLocalHost; + +class CNodeStats +{ +public: + NodeId nodeid; + ServiceFlags nServices; + bool fRelayTxes; + int64_t nLastSend; + int64_t nLastRecv; + int64_t nTimeConnected; + int64_t nTimeOffset; + std::string addrName; + int nVersion; + std::string cleanSubVer; + bool fInbound; + bool fAddnode; + int nStartingHeight; + uint64_t nSendBytes; + mapMsgCmdSize mapSendBytesPerMsgCmd; + uint64_t nRecvBytes; + mapMsgCmdSize mapRecvBytesPerMsgCmd; + bool fWhitelisted; + double dPingTime; + double dPingWait; + double dMinPing; + std::string addrLocal; + CAddress addr; +}; + +// Exception-safe class for holding a reference to a CNode +class CNodeRef +{ + void AddRef() { - LOCK(cs_inventory); - filterInventoryKnown.insert(inv.hash); + if (_pnode) + _pnode->AddRef(); } - void PushInventory(const CInv &inv) + void Release() { - LOCK(cs_inventory); - if (inv.type == MSG_TX) - { - if (!filterInventoryKnown.contains(inv.hash)) - { - setInventoryTxToSend.insert(inv.hash); - } - } - else if (inv.type == MSG_BLOCK) + if (_pnode) { - vInventoryBlockToSend.push_back(inv.hash); + // Make the noderef null before releasing, to ensure a user can't get freed memory from us + CNode *tmp = _pnode; + _pnode = nullptr; + tmp->Release(); } } - void PushBlockHash(const uint256 &hash) +public: + CNodeRef(CNode *pnode = nullptr) : _pnode(pnode) { AddRef(); } + CNodeRef(const CNodeRef &other) : _pnode(other._pnode) { AddRef(); } + ~CNodeRef() { Release(); } + CNode &operator*() const { return *_pnode; }; + CNode *operator->() const { return _pnode; }; + // Returns true if this reference is not null + explicit operator bool() const { return _pnode; } + // Access the raw pointer + CNode *get() const { return _pnode; } + // Assignment -- destroys any reference to the current node and adds a ref to the new one + CNodeRef &operator=(CNode *pnode) { - LOCK(cs_inventory); - vBlockHashesToAnnounce.push_back(hash); + if (pnode != _pnode) + { + Release(); + _pnode = pnode; + AddRef(); + } + return *this; } + // Assignment -- destroys any reference to the current node and adds a ref to the new one + CNodeRef &operator=(const CNodeRef &other) { return operator=(other._pnode); } +private: + CNode *_pnode; +}; - void AskFor(const CInv &inv); - - void CloseSocketDisconnect(); - - void copyStats(CNodeStats &stats); +// Connection Slot mitigation - used to track connection attempts and evictions +struct ConnectionHistory +{ + double nConnections; // number of connection attempts made within 1 minute + int64_t nLastConnectionTime; // the time the last connection attempt was made - ServiceFlags GetLocalServices() const { return nLocalServices; } - std::string GetAddrName() const; - //! Sets the addrName only if it was not previously set - void MaybeSetAddrName(const std::string &addrNameIn); + double nEvictions; // number of times a connection was de-prioritized and disconnected in last 30 minutes + int64_t nLastEvictionTime; // the time the last eviction occurred. }; /** diff --git a/src/netaddress.cpp b/src/net/netaddress.cpp similarity index 100% rename from src/netaddress.cpp rename to src/net/netaddress.cpp diff --git a/src/netaddress.h b/src/net/netaddress.h similarity index 100% rename from src/netaddress.h rename to src/net/netaddress.h diff --git a/src/netbase.cpp b/src/net/netbase.cpp similarity index 99% rename from src/netbase.cpp rename to src/net/netbase.cpp index f7d5e75a..51cec03c 100644 --- a/src/netbase.cpp +++ b/src/net/netbase.cpp @@ -24,7 +24,7 @@ #include -#include "netbase.h" +#include "net/netbase.h" #include "crypto/hash.h" #include "random.h" diff --git a/src/netbase.h b/src/net/netbase.h similarity index 100% rename from src/netbase.h rename to src/net/netbase.h diff --git a/src/protocol.cpp b/src/net/protocol.cpp similarity index 99% rename from src/protocol.cpp rename to src/net/protocol.cpp index e289abc8..e00e0052 100644 --- a/src/protocol.cpp +++ b/src/net/protocol.cpp @@ -18,7 +18,7 @@ * along with this program. If not, see . */ -#include "protocol.h" +#include "net/protocol.h" #include "util/util.h" #include "util/utilstrencodings.h" diff --git a/src/protocol.h b/src/net/protocol.h similarity index 99% rename from src/protocol.h rename to src/net/protocol.h index 60374b48..6ea60e8c 100644 --- a/src/protocol.h +++ b/src/net/protocol.h @@ -25,7 +25,7 @@ #ifndef BITCOIN_PROTOCOL_H #define BITCOIN_PROTOCOL_H -#include "netbase.h" +#include "net/netbase.h" #include "serialize.h" #include "uint256.h" #include "version.h" diff --git a/src/netmessagemaker.h b/src/netmessagemaker.h deleted file mode 100644 index a0e648af..00000000 --- a/src/netmessagemaker.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2009-2010 Satoshi Nakamoto - * Copyright (c) 2009-2016 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef BITCOIN_NETMESSAGEMAKER_H -#define BITCOIN_NETMESSAGEMAKER_H - -#include "net.h" -#include "serialize.h" - -class CNetMsgMaker -{ -public: - CNetMsgMaker(int nVersionIn) : nVersion(nVersionIn) {} - template - CSerializedNetMsg Make(int nFlags, std::string sCommand, Args &&... args) const - { - CSerializedNetMsg msg; - msg.command = std::move(sCommand); - CVectorWriter{SER_NETWORK, nFlags | nVersion, msg.data, 0, std::forward(args)...}; - return msg; - } - - template - CSerializedNetMsg Make(std::string sCommand, Args &&... args) const - { - return Make(0, std::move(sCommand), std::forward(args)...); - } - -private: - const int nVersion; -}; - -#endif // BITCOIN_NETMESSAGEMAKER_H diff --git a/src/networks/networktemplate.h b/src/networks/networktemplate.h index bfd755be..fd1a4b7c 100644 --- a/src/networks/networktemplate.h +++ b/src/networks/networktemplate.h @@ -21,7 +21,7 @@ #include "chain/block.h" #include "consensus/params.h" -#include "protocol.h" +#include "net/protocol.h" #include diff --git a/src/processblock.cpp b/src/processblock.cpp index 3e5f9d37..28f526a4 100644 --- a/src/processblock.cpp +++ b/src/processblock.cpp @@ -34,8 +34,8 @@ #include "init.h" #include "kernel.h" #include "main.h" -#include "messages.h" -#include "net.h" +#include "net/messages.h" +#include "net/net.h" #include "networks/netman.h" #include "networks/networktemplate.h" #include "policy/policy.h" @@ -752,7 +752,6 @@ bool ActivateBestChain(CValidationState &state, } } }); - g_connman->WakeMessageHandler(); // Notify external listeners about the new tip. if (!vHashes.empty()) { diff --git a/src/processtx.cpp b/src/processtx.cpp index 45488238..fdb42b6d 100644 --- a/src/processtx.cpp +++ b/src/processtx.cpp @@ -21,7 +21,7 @@ #include "base58.h" #include "consensus/consensus.h" #include "main.h" -#include "messages.h" +#include "net/messages.h" #include "script/standard.h" bool CheckTransaction(const CTransaction &tx, CValidationState &state) diff --git a/src/rpc/rpcmining.cpp b/src/rpc/rpcmining.cpp index 13489cc6..80a2341d 100644 --- a/src/rpc/rpcmining.cpp +++ b/src/rpc/rpcmining.cpp @@ -28,7 +28,7 @@ #include "core_io.h" #include "init.h" #include "main.h" -#include "net.h" +#include "net/net.h" #include "networks/netman.h" #include "pow.h" #include "processblock.h" diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index 99457302..0a9115df 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -24,8 +24,8 @@ #include "clientversion.h" #include "init.h" #include "main.h" -#include "net.h" -#include "netbase.h" +#include "net/net.h" +#include "net/netbase.h" #include "rpcserver.h" #include "timedata.h" #include "util/util.h" diff --git a/src/rpc/rpcnet.cpp b/src/rpc/rpcnet.cpp index 971a1712..1972cbef 100644 --- a/src/rpc/rpcnet.cpp +++ b/src/rpc/rpcnet.cpp @@ -22,11 +22,11 @@ #include "clientversion.h" #include "main.h" -#include "messages.h" -#include "net.h" -#include "netbase.h" +#include "net/messages.h" +#include "net/net.h" +#include "net/netbase.h" +#include "net/protocol.h" #include "networks/netman.h" -#include "protocol.h" #include "sync.h" #include "timedata.h" #include "ui_interface.h" @@ -463,7 +463,6 @@ UniValue getnetworkinfo(const UniValue ¶ms, bool fHelp) obj.push_back(Pair("timeoffset", GetTimeOffset())); if (g_connman) { - obj.push_back(Pair("networkactive", g_connman->GetNetworkActive())); obj.push_back(Pair("connections", (int)g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL))); } obj.push_back(Pair("networks", GetNetworksInfo())); diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 0478b3e4..5c911919 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -28,7 +28,7 @@ #include "keystore.h" #include "main.h" #include "merkleblock.h" -#include "net.h" +#include "net/net.h" #include "policy/policy.h" #include "rpcserver.h" #include "script/script.h" diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 0c4cefeb..64f53440 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -24,8 +24,8 @@ #include "core_io.h" #include "init.h" #include "main.h" -#include "net.h" -#include "netbase.h" +#include "net/net.h" +#include "net/netbase.h" #include "rpcserver.h" #include "timedata.h" #include "util/util.h" diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index a12958c7..19fb78bb 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -8,7 +8,7 @@ #include "dosman.h" #include "keystore.h" #include "main.h" -#include "net.h" +#include "net/net.h" #include "pow.h" #include "script/sign.h" #include "serialize.h" diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 625a1cda..41ec43d8 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -2,7 +2,7 @@ // Copyright (c) 2015-2017 The Bitcoin Unlimited developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrman.h" +#include "net/addrman.h" #include "test/test_bitcoin.h" #include #include diff --git a/src/test/exploit_tests.cpp b/src/test/exploit_tests.cpp index bd58b26f..dd5b34e1 100644 --- a/src/test/exploit_tests.cpp +++ b/src/test/exploit_tests.cpp @@ -6,8 +6,8 @@ #include "chain/block.h" #include "main.h" -#include "net.h" -#include "protocol.h" +#include "net/net.h" +#include "net/protocol.h" #include "random.h" #include "requestManager.h" #include "serialize.h" diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 76e17c58..d998a27f 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1,10 +1,10 @@ // Copyright (c) 2012-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "net.h" -#include "addrman.h" +#include "net/net.h" #include "crypto/hash.h" -#include "netbase.h" +#include "net/addrman.h" +#include "net/netbase.h" #include "serialize.h" #include "streams.h" #include "test/test_bitcoin.h" diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index b6414c34..446ba906 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "netbase.h" +#include "net/netbase.h" #include "test/test_bitcoin.h" #include "util/utilstrencodings.h" diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 64db0693..87a556ae 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -6,7 +6,7 @@ #include "rpc/rpcserver.h" #include "base58.h" -#include "netbase.h" +#include "net/netbase.h" #include "test/test_bitcoin.h" @@ -98,7 +98,6 @@ BOOST_AUTO_TEST_CASE(rpc_togglenetwork) bool netState = find_value(r.get_obj(), "networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, true); - BOOST_CHECK_NO_THROW(CallRPC("setnetworkactive false")); r = CallRPC("getnetworkinfo"); int numConnection = find_value(r.get_obj(), "connections").get_int(); BOOST_CHECK_EQUAL(numConnection, 0); @@ -106,7 +105,6 @@ BOOST_AUTO_TEST_CASE(rpc_togglenetwork) netState = find_value(r.get_obj(), "networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, false); - BOOST_CHECK_NO_THROW(CallRPC("setnetworkactive true")); r = CallRPC("getnetworkinfo"); netState = find_value(r.get_obj(), "networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, true); diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 91c32386..75dc86a4 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -16,7 +16,7 @@ #include "init.h" #include "key.h" #include "main.h" -#include "messages.h" +#include "net/messages.h" #include "processblock.h" #include "pubkey.h" #include "random.h" diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index 07632fd4..dbc95532 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -6,13 +6,13 @@ #include "config/bitcoin-config.h" #endif -#include "addrman.h" #include "chain/chain.h" #include "coins.h" #include "compressor.h" #include "consensus/merkle.h" -#include "net.h" -#include "protocol.h" +#include "net/addrman.h" +#include "net/net.h" +#include "net/protocol.h" #include "pubkey.h" #include "script/script.h" #include "streams.h" diff --git a/src/threadgroup.h b/src/threadgroup.h index a1061890..38a4c422 100644 --- a/src/threadgroup.h +++ b/src/threadgroup.h @@ -13,9 +13,11 @@ class thread_group private: std::vector threads; std::vector names; + std::atomic *killswitch; public: - void interrupt_all() { shutdown_threads.store(true); } + thread_group(std::atomic *_killswitch) { killswitch = _killswitch; } + void interrupt_all() { killswitch->store(true); } template void create_thread(std::string name, Fn &&f, Args &&... args) { @@ -23,6 +25,14 @@ class thread_group names.push_back(name); } + void clear() + { + interrupt_all(); + join_all(); + threads.clear(); + names.clear(); + } + bool empty() { return threads.empty(); } void join_all() { diff --git a/src/timedata.cpp b/src/timedata.cpp index 331824a1..def86da7 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -19,7 +19,7 @@ #include "timedata.h" -#include "netbase.h" +#include "net/netbase.h" #include "sync.h" #include "ui_interface.h" #include "util/util.h" diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 46b07594..ae24f521 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -21,7 +21,7 @@ #include "torcontrol.h" #include "args.h" #include "crypto/hmac_sha256.h" -#include "net.h" +#include "net/net.h" #include "util/util.h" #include "util/utilstrencodings.h" diff --git a/src/txdb.cpp b/src/txdb.cpp index 56a123fc..a0602afb 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -369,8 +369,7 @@ bool CCoinsViewDB::Upgrade() return true; } - LogPrintf("Upgrading database...\n"); - uiInterface.InitMessage(_("Upgrading database...this may take a while")); + LogPrintf("Upgrading database...this may take a while\n"); size_t batch_size = 1 << 24; CDBBatch batch(db); diff --git a/src/version.h b/src/version.h index 48fc040d..03e95b28 100644 --- a/src/version.h +++ b/src/version.h @@ -44,9 +44,12 @@ static const int NO_BLOOM_VERSION = 60034; * Versioning for network services */ -// version of the service transaction resolution code -static const int NETWORK_SERVICE_VERSION = 10003; +#define NETWORK_SERVICE_VERSION_MAJOR 0 +#define NETWORK_SERVICE_VERSION_MINOR 1 +#define NETWORK_SERVICE_VERSION_REVISION 0 -static const int ANS_VERSION = 10000; +// version of the service transaction resolution code +static const int NETWORK_SERVICE_VERSION = + 10000 * NETWORK_SERVICE_VERSION_MAJOR + 100 * NETWORK_SERVICE_VERSION_MINOR + 1 * NETWORK_SERVICE_VERSION_REVISION; #endif // BITCOIN_VERSION_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index f7583c04..5b42623f 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -20,10 +20,10 @@ #include "db.h" -#include "addrman.h" #include "args.h" #include "crypto/hash.h" -#include "protocol.h" +#include "net/addrman.h" +#include "net/protocol.h" #include "threadgroup.h" #include "util/util.h" #include "util/utilstrencodings.h" diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 318b331a..c850d305 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -33,7 +33,7 @@ #include "key.h" #include "keystore.h" #include "main.h" -#include "net.h" +#include "net/net.h" #include "policy/policy.h" #include "script/script.h" #include "script/sign.h" @@ -3668,7 +3668,7 @@ bool CWallet::InitLoadWallet() if (gArgs.GetBoolArg("-zapwallettxes", false)) { - uiInterface.InitMessage(_("Zapping all transactions from wallet...")); + LogPrintf("Zapping all transactions from wallet..."); CWallet *tempWallet = new CWallet(walletFile); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); @@ -3681,7 +3681,7 @@ bool CWallet::InitLoadWallet() tempWallet = nullptr; } - uiInterface.InitMessage(_("Loading wallet...")); + LogPrintf("Loading wallet..."); int64_t nStart = GetTimeMillis(); bool fFirstRun = true; @@ -3724,7 +3724,9 @@ bool CWallet::InitLoadWallet() walletInstance->SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately } else + { LogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + } if (nMaxVersion < walletInstance->GetVersion()) { return UIError(_("Cannot downgrade wallet")); @@ -3764,7 +3766,6 @@ bool CWallet::InitLoadWallet() } if (pnetMan->getChainActive()->chainActive.Tip() && pnetMan->getChainActive()->chainActive.Tip() != pindexRescan) { - uiInterface.InitMessage(_("Rescanning...")); LogPrintf("Rescanning last %i blocks (from block %i)...\n", pnetMan->getChainActive()->chainActive.Height() - pindexRescan->nHeight, pindexRescan->nHeight); nStart = GetTimeMillis(); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 2e5c5d53..ad37402f 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -23,7 +23,7 @@ #include "amount.h" #include "consensus/consensus.h" -#include "net.h" +#include "net/net.h" #include "policy/policy.h" #include "streams.h" #include "tinyformat.h" diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 2e7b77d1..7c374edf 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -23,8 +23,8 @@ #include "args.h" #include "base58.h" #include "consensus/validation.h" +#include "net/protocol.h" #include "processtx.h" // For CheckTransaction -#include "protocol.h" #include "serialize.h" #include "sync.h" #include "util/util.h" From 2780ea154711481df8b4cc1f3b5ef35004dc3938 Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Mon, 28 Jan 2019 04:39:36 -0500 Subject: [PATCH 05/46] update help menu options and some policy defaults (#122) --- src/Makefile.am | 4 +- src/bitcoin-tx-res.rc | 35 ------------- src/blockgeneration/miner.cpp | 17 ++----- src/blockgeneration/minter.cpp | 17 ++----- src/consensus/consensus.h | 9 ++-- src/{ => crypto}/pbkdf2.cpp | 0 src/{ => crypto}/pbkdf2.h | 0 src/init.cpp | 91 ++++++---------------------------- src/main.cpp | 11 +--- src/main.h | 9 +--- src/net/messages.cpp | 2 +- src/policy/policy.h | 5 -- src/rpc/rpcprotocol.cpp | 3 -- src/ui_interface.h | 6 --- src/wallet/wallet.cpp | 2 +- src/wallet/wallet.h | 2 +- 16 files changed, 32 insertions(+), 181 deletions(-) delete mode 100644 src/bitcoin-tx-res.rc rename src/{ => crypto}/pbkdf2.cpp (100%) rename src/{ => crypto}/pbkdf2.h (100%) diff --git a/src/Makefile.am b/src/Makefile.am index e9a5c121..a9ef0f30 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -98,7 +98,7 @@ BITCOIN_CORE_H = \ prevector.h \ chain/block.h \ chain/tx.h \ - pbkdf2.h \ + crypto/pbkdf2.h \ processblock.h \ processheader.h \ processtx.h \ @@ -216,7 +216,7 @@ libbitcoin_server_a_SOURCES = \ chain/tx.cpp \ net/protocol.cpp \ pubkey.cpp \ - pbkdf2.cpp \ + crypto/pbkdf2.cpp \ script/interpreter.cpp \ script/script.cpp \ script/script_error.cpp \ diff --git a/src/bitcoin-tx-res.rc b/src/bitcoin-tx-res.rc deleted file mode 100644 index 3e49b820..00000000 --- a/src/bitcoin-tx-res.rc +++ /dev/null @@ -1,35 +0,0 @@ -#include // needed for VERSIONINFO -#include "clientversion.h" // holds the needed client version information - -#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_REVISION,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) -#define VER_FILEVERSION VER_PRODUCTVERSION -#define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR - -VS_VERSION_INFO VERSIONINFO -FILEVERSION VER_FILEVERSION -PRODUCTVERSION VER_PRODUCTVERSION -FILEOS VOS_NT_WINDOWS32 -FILETYPE VFT_APP -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904E4" // U.S. English - multilingual (hex) - BEGIN - VALUE "CompanyName", "Bitcoin" - VALUE "FileDescription", "bitcoin-tx (CLI Bitcoin transaction editor utility)" - VALUE "FileVersion", VER_FILEVERSION_STR - VALUE "InternalName", "bitcoin-tx" - VALUE "LegalCopyright", COPYRIGHT_STR - VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." - VALUE "OriginalFilename", "bitcoin-tx.exe" - VALUE "ProductName", "bitcoin-tx" - VALUE "ProductVersion", VER_PRODUCTVERSION_STR - END - END - - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x0, 1252 // language neutral - multilingual (decimal) - END -END diff --git a/src/blockgeneration/miner.cpp b/src/blockgeneration/miner.cpp index 2032b74a..e7d17375 100644 --- a/src/blockgeneration/miner.cpp +++ b/src/blockgeneration/miner.cpp @@ -131,20 +131,9 @@ std::unique_ptr CreateNewPoWBlock(CWallet *pwallet, const CScrip pblocktemplate->vTxFees.push_back(-1); // updated at end pblocktemplate->vTxSigOps.push_back(-1); // updated at end - // Largest block you're willing to create: - unsigned int nBlockMaxSize = gArgs.GetArg("-blockmaxsize", MAX_BLOCK_SIZE_GEN / 2); - // Limit to betweeen 1K and MAX_BLOCK_SIZE-1K for sanity: - nBlockMaxSize = std::max((unsigned int)1000, std::min((unsigned int)(MAX_BLOCK_SIZE - 1000), nBlockMaxSize)); - - // How much of the block should be dedicated to high-priority transactions, - // included regardless of the fees they pay - unsigned int nBlockPrioritySize = gArgs.GetArg("-blockprioritysize", 27000); - nBlockPrioritySize = std::min(nBlockMaxSize, nBlockPrioritySize); - - // Minimum block size you want to create; block will be filled with free transactions - // until there are no more or the block reaches this size: - unsigned int nBlockMinSize = gArgs.GetArg("-blockminsize", 0); - nBlockMinSize = std::min(nBlockMaxSize, nBlockMinSize); + uint64_t nBlockMaxSize = MAX_BLOCK_SIZE - 1000; + uint64_t nBlockPrioritySize = DEFTAUL_BLOCK_PRIORITY_SIZE; + uint64_t nBlockMinSize = 0; // Collect memory pool transactions into the block CTxMemPool::setEntries inBlock; diff --git a/src/blockgeneration/minter.cpp b/src/blockgeneration/minter.cpp index 680125a4..07ec50c3 100644 --- a/src/blockgeneration/minter.cpp +++ b/src/blockgeneration/minter.cpp @@ -97,20 +97,9 @@ std::unique_ptr CreateNewPoSBlock(CWallet *pwallet, const CScrip pblocktemplate->vTxFees.push_back(-1); // updated at end pblocktemplate->vTxSigOps.push_back(-1); // updated at end - // Largest block you're willing to create: - unsigned int nBlockMaxSize = gArgs.GetArg("-blockmaxsize", MAX_BLOCK_SIZE_GEN / 2); - // Limit to betweeen 1K and MAX_BLOCK_SIZE-1K for sanity: - nBlockMaxSize = std::max((unsigned int)1000, std::min((unsigned int)(MAX_BLOCK_SIZE - 1000), nBlockMaxSize)); - - // How much of the block should be dedicated to high-priority transactions, - // included regardless of the fees they pay - unsigned int nBlockPrioritySize = gArgs.GetArg("-blockprioritysize", 27000); - nBlockPrioritySize = std::min(nBlockMaxSize, nBlockPrioritySize); - - // Minimum block size you want to create; block will be filled with free transactions - // until there are no more or the block reaches this size: - unsigned int nBlockMinSize = gArgs.GetArg("-blockminsize", 0); - nBlockMinSize = std::min(nBlockMaxSize, nBlockMinSize); + uint64_t nBlockMaxSize = MAX_BLOCK_SIZE - 1000; + uint64_t nBlockPrioritySize = DEFTAUL_BLOCK_PRIORITY_SIZE; + uint64_t nBlockMinSize = 0; // Collect memory pool transactions into the block CTxMemPool::setEntries inBlock; diff --git a/src/consensus/consensus.h b/src/consensus/consensus.h index 109d70d2..77100ec5 100644 --- a/src/consensus/consensus.h +++ b/src/consensus/consensus.h @@ -23,13 +23,10 @@ #include - /** The maximum allowed size for a serialized block, in bytes (network rule) */ -static const uint64_t MAX_BLOCK_SIZE = 1000000; - -static const uint64_t MAX_BLOCK_SIZE_GEN = MAX_BLOCK_SIZE / 2; - -static const uint64_t DEFAULT_LARGEST_TRANSACTION = 1000000; +static const uint64_t MAX_BLOCK_SIZE = 1000000; // 1MB +static const uint64_t DEFTAUL_BLOCK_PRIORITY_SIZE = 100000; // 0.1MB +static const uint64_t DEFAULT_LARGEST_TRANSACTION = 1000000; // 1MB static const int64_t nMaxClockDrift = 2 * 60 * 60; // two hours diff --git a/src/pbkdf2.cpp b/src/crypto/pbkdf2.cpp similarity index 100% rename from src/pbkdf2.cpp rename to src/crypto/pbkdf2.cpp diff --git a/src/pbkdf2.h b/src/crypto/pbkdf2.h similarity index 100% rename from src/pbkdf2.h rename to src/crypto/pbkdf2.h diff --git a/src/init.cpp b/src/init.cpp index bc5183ff..5745369c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -340,10 +340,6 @@ std::string HelpMessage() std::string strUsage = HelpMessageGroup(_("Options:")); strUsage += HelpMessageOpt("-?", _("This help message")); strUsage += HelpMessageOpt("-version", _("Print version and exit")); - strUsage += - HelpMessageOpt("-alerts", strprintf(_("Receive and display P2P network alerts (default: %u)"), DEFAULT_ALERTS)); - strUsage += HelpMessageOpt("-alertnotify=", _("Execute command when a relevant alert is received or we see a " - "really long fork (%s in cmd is replaced by message)")); strUsage += HelpMessageOpt( "-blocknotify=", _("Execute command when the best block changes (%s in cmd is replaced by block hash)")); strUsage += HelpMessageOpt("-checkblocks=", @@ -378,10 +374,6 @@ std::string HelpMessage() strUsage += HelpMessageOpt("-pid=", strprintf(_("Specify pid file (default: %s)"), PID_FILENAME)); #endif strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files on startup")); -#ifndef WIN32 - strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 " - "(only effective with disabled wallet functionality)")); -#endif strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); @@ -418,9 +410,6 @@ std::string HelpMessage() "-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %u)"), DEFAULT_PERMIT_BAREMULTISIG)); strUsage += HelpMessageOpt("-peerbloomfilters", strprintf(_("Support filtering of blocks and transaction with bloom filters (default: %u)"), 1)); - if (showDebug) - strUsage += HelpMessageOpt("-enforcenodebloom", - strprintf("Enforce minimum protocol version to limit use of bloom filters (default: %u)", 0)); strUsage += HelpMessageOpt("-port=", strprintf(_("Listen for connections on (default: %u)"), pnetMan->getActivePaymentNetwork()->GetDefaultPort())); strUsage += HelpMessageOpt("-proxy=", _("Connect through SOCKS5 proxy")); @@ -466,7 +455,6 @@ std::string HelpMessage() strUsage += HelpMessageGroup(_("Wallet options:")); - strUsage += HelpMessageOpt("-disablewallet", _("Do not load the wallet and disable wallet RPC calls")); strUsage += HelpMessageOpt("-keypool=", strprintf(_("Set key pool size to (default: %u)"), DEFAULT_KEYPOOL_SIZE)); strUsage += HelpMessageOpt("-fallbackfee=", @@ -535,6 +523,7 @@ std::string HelpMessage() "-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET)); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT)); + strUsage += HelpMessageOpt("-limitancestorcount=", strprintf("Do not accept transactions if number of in-mempool ancestors is or more (default: %u)", DEFAULT_ANCESTOR_LIMIT)); @@ -575,9 +564,6 @@ std::string HelpMessage() strUsage += HelpMessageOpt("-limitfreerelay=", strprintf("Continuously rate-limit free transactions to *1000 bytes per minute (default: %u)", DEFAULT_LIMITFREERELAY)); - strUsage += HelpMessageOpt( - "-relaypriority", strprintf("Require high priority for relaying free or low-fee transactions (default: %u)", - DEFAULT_RELAYPRIORITY)); strUsage += HelpMessageOpt("-maxsigcachesize=", strprintf("Limit size of signature cache to MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE)); } @@ -595,8 +581,6 @@ std::string HelpMessage() strUsage += HelpMessageOpt("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB)); } - strUsage += - HelpMessageOpt("-shrinkdebugfile", _("Shrink debug.log file on client startup (default: 1 when no -debug)")); strUsage += HelpMessageGroup(_("Node relay options:")); @@ -614,17 +598,8 @@ std::string HelpMessage() strUsage += HelpMessageOpt("-datacarriersize", strprintf(_("Maximum size of data in data carrier transactions we relay and mine (default: %u)"), MAX_OP_RETURN_RELAY)); - strUsage += HelpMessageOpt("-mempoolreplacement", - strprintf(_("Enable transaction replacement in the memory pool (default: %u)"), DEFAULT_ENABLE_REPLACEMENT)); strUsage += HelpMessageGroup(_("Block creation options:")); - strUsage += HelpMessageOpt( - "-blockminsize=", strprintf(_("Set minimum block size in bytes (default: %u)"), DEFAULT_BLOCK_MIN_SIZE)); - strUsage += HelpMessageOpt( - "-blockmaxsize=", strprintf(_("Set maximum block size in bytes (default: %d)"), DEFAULT_BLOCK_MAX_SIZE)); - strUsage += HelpMessageOpt("-blockprioritysize=", - strprintf(_("Set maximum size of high-priority/low-fee transactions in bytes (default: %d)"), - DEFAULT_BLOCK_PRIORITY_SIZE)); if (showDebug) strUsage += HelpMessageOpt("-blockversion=", "Override block version to test forking scenarios"); @@ -945,15 +920,7 @@ bool AppInit2(thread_group &threadGroup) return InitError("Initializing networking failed"); #ifndef WIN32 - if (gArgs.GetBoolArg("-sysperms", false)) - { - if (!gArgs.GetBoolArg("-disablewallet", false)) - return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); - } - else - { - umask(077); - } + umask(077); // Clean shutdown on SIGTERM struct sigaction sa; @@ -1056,8 +1023,6 @@ bool AppInit2(thread_group &threadGroup) fServer = gArgs.GetBoolArg("-server", false); - bool fDisableWallet = gArgs.GetBoolArg("-disablewallet", false); - nConnectTimeout = gArgs.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; @@ -1152,22 +1117,9 @@ bool AppInit2(thread_group &threadGroup) fAcceptDatacarrier = gArgs.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); nMaxDatacarrierBytes = gArgs.GetArg("-datacarriersize", nMaxDatacarrierBytes); - fAlerts = gArgs.GetBoolArg("-alerts", DEFAULT_ALERTS); - // Option to startup with mocktime set (used for regression testing): SetMockTime(gArgs.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op - fEnableReplacement = gArgs.GetBoolArg("-mempoolreplacement", DEFAULT_ENABLE_REPLACEMENT); - if ((!fEnableReplacement) && gArgs.IsArgSet("-mempoolreplacement")) - { - // Minimal effort at forwards compatibility - std::string strReplacementModeList = gArgs.GetArg("-mempoolreplacement", ""); // default is impossible - std::vector vstrReplacementModes; - boost::split(vstrReplacementModes, strReplacementModeList, boost::is_any_of(",")); - fEnableReplacement = - (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end()); - } - // ********************************************************* Step 4: application initialization: dir lock, // daemonize, pidfile, debug log @@ -1207,8 +1159,6 @@ bool AppInit2(thread_group &threadGroup) #ifndef WIN32 CreatePidFile(GetPidFile(), getpid()); #endif - if (gArgs.GetBoolArg("-shrinkdebugfile", !fDebug)) - ShrinkDebugFile(); if (fPrintToDebugLog) OpenDebugLog(); @@ -1249,23 +1199,20 @@ bool AppInit2(thread_group &threadGroup) // ********************************************************* Step 5: verify wallet database integrity - if (!fDisableWallet) - { - LogPrintf("Using wallet %s\n", strWalletFile); - LogPrintf("Verifying wallet..."); - std::string warningString; - std::string errorString; + LogPrintf("Using wallet %s\n", strWalletFile); + uiInterface.InitMessage(_("Verifying wallet...")); - if (!CWallet::Verify(strWalletFile, warningString, errorString)) - return false; + std::string warningString; + std::string errorString; - if (!warningString.empty()) - InitWarning(warningString); - if (!errorString.empty()) - return InitError(errorString); + if (!CWallet::Verify(strWalletFile, warningString, errorString)) + return false; + if (!warningString.empty()) + InitWarning(warningString); + if (!errorString.empty()) + return InitError(errorString); - } // (!fDisableWallet) // ********************************************************* Step 6: network initialization assert(!g_connman); @@ -1639,17 +1586,9 @@ bool AppInit2(thread_group &threadGroup) // ********************************************************* Step 8: load wallet - if (fDisableWallet) - { - pwalletMain = NULL; - LogPrintf("Wallet disabled!\n"); - } - else - { - CWallet::InitLoadWallet(); - if (!pwalletMain) - return false; - } + CWallet::InitLoadWallet(); + if (!pwalletMain) + return false; // ********************************************************* Step 10: import blocks diff --git a/src/main.cpp b/src/main.cpp index 3289b26b..8a5e2fca 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -88,8 +88,6 @@ unsigned int nBytesPerSigOp = DEFAULT_BYTES_PER_SIGOP; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; size_t nCoinCacheUsage = 5000 * 300; -bool fAlerts = DEFAULT_ALERTS; -bool fEnableReplacement = DEFAULT_ENABLE_REPLACEMENT; /** Fees smaller than this (in satoshi) are considered zero fee (for relaying, mining and transaction creation) */ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); @@ -658,13 +656,6 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nFees, mempoolRejectFee)); } - else if (gArgs.GetBoolArg("-relaypriority", DEFAULT_RELAYPRIORITY) && - nModifiedFees < ::minRelayTxFee.GetFee(nSize) && - !AllowFree(entry.GetPriority(pnetMan->getChainActive()->chainActive.Height() + 1))) - { - // Require that free transactions have sufficient priority to be mined in the next block. - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority"); - } // Continuously rate-limit free (really, very-low-fee) transactions // This mitigates 'penny-flooding' -- sending thousands of free transactions just to @@ -744,7 +735,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, // -limitfreerelay unit is thousand-bytes-per-minute // At default rate it would take over a month to fill 1GB if (dFreeCount >= gArgs.GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) * 10 * 1000) + { return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "rate limited free transaction"); + } LogPrint("mempool", "Rate limit dFreeCount: %g => %g\n", dFreeCount, dFreeCount + nSize); dFreeCount += nSize; } diff --git a/src/main.h b/src/main.h index 029e9d3d..9812f823 100644 --- a/src/main.h +++ b/src/main.h @@ -56,8 +56,6 @@ struct LockPoints; /** Default for returning change from tx back an address we already owned instead of a new one (try to select address * with most value in it). */ static const bool DEFAULT_RETURN_CHANGE = true; -/** Default for accepting alerts from the P2P network. */ -static const bool DEFAULT_ALERTS = false; /** Default for DEFAULT_WHITELISTRELAY. */ static const bool DEFAULT_WHITELISTRELAY = true; /** Default for DEFAULT_WHITELISTFORCERELAY. */ @@ -117,8 +115,7 @@ static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; /** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; -static const unsigned int DEFAULT_LIMITFREERELAY = 15; -static const bool DEFAULT_RELAYPRIORITY = true; +static const unsigned int DEFAULT_LIMITFREERELAY = 1000; /** Default for -permitbaremultisig */ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; @@ -128,8 +125,6 @@ static const bool DEFAULT_TXINDEX = true; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; -/** Default for -mempoolreplacement */ -static const bool DEFAULT_ENABLE_REPLACEMENT = true; /** The minimum value possible for -limitfreerelay when rate limiting */ static const unsigned int DEFAULT_MIN_LIMITFREERELAY = 1; /** The default value for -minrelaytxfee */ @@ -164,8 +159,6 @@ extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; extern size_t nCoinCacheUsage; extern CFeeRate minRelayTxFee; -extern bool fAlerts; -extern bool fEnableReplacement; struct COrphanTx { diff --git a/src/net/messages.cpp b/src/net/messages.cpp index 163f6f59..5ec03ebc 100644 --- a/src/net/messages.cpp +++ b/src/net/messages.cpp @@ -1124,7 +1124,7 @@ bool static ProcessMessage(CNode *pfrom, Misbehaving(pfrom->GetId(), 100, "no-bloom-version"); return false; } - else if (gArgs.GetBoolArg("-enforcenodebloom", false)) + else { pfrom->fDisconnect = true; return false; diff --git a/src/policy/policy.h b/src/policy/policy.h index 589ff266..85b7955f 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -29,11 +29,6 @@ class CCoinsViewCache; -/** Default for -blockmaxsize and -blockminsize, which control the range of sizes the mining code will create **/ -static const unsigned int DEFAULT_BLOCK_MAX_SIZE = 750000; -static const unsigned int DEFAULT_BLOCK_MIN_SIZE = 0; -/** Default for -blockprioritysize, maximum space for zero/low-fee transactions **/ -static const unsigned int DEFAULT_BLOCK_PRIORITY_SIZE = 0; /** The maximum size for transactions we're willing to relay/mine */ static const unsigned int MAX_STANDARD_TX_SIZE = 100000; /** Maximum number of signature check operations in an IsStandard() P2SH script */ diff --git a/src/rpc/rpcprotocol.cpp b/src/rpc/rpcprotocol.cpp index 83e199f5..f8ffd917 100644 --- a/src/rpc/rpcprotocol.cpp +++ b/src/rpc/rpcprotocol.cpp @@ -105,9 +105,6 @@ bool GenerateAuthCookie(std::string *cookie_out) GetRandBytes(rand_pwd, 32); std::string cookie = COOKIEAUTH_USER + ":" + EncodeBase64(&rand_pwd[0], 32); - /** the umask determines what permissions are used to create this file - - * these are set to 077 in init.cpp unless overridden with -sysperms. - */ std::ofstream file; fs::path filepath = GetAuthCookieFile(); file.open(filepath.string().c_str()); diff --git a/src/ui_interface.h b/src/ui_interface.h index 2d7166a7..0c4b841d 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -102,12 +102,6 @@ class CClientUIInterface /** Number of network connections changed. */ boost::signals2::signal NotifyNumConnectionsChanged; - /** - * New, updated or cancelled alert. - * @note called with lock cs_mapAlerts held. - */ - boost::signals2::signal NotifyAlertChanged; - /** A wallet has been loaded. */ boost::signals2::signal LoadWallet; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index c850d305..dea21c85 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3640,7 +3640,7 @@ bool CWallet::CreateCoinStake(const CKeyStore &keystore, // Limit size unsigned int nBytes = ::GetSerializeSize(txNew, SER_NETWORK, PROTOCOL_VERSION); - if (nBytes >= MAX_BLOCK_SIZE_GEN / 5) + if (nBytes >= MAX_BLOCK_SIZE / 10) return error("CreateCoinStake : exceeded coinstake size limit"); // Successfully generated coinstake diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index ad37402f..c8456746 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -61,7 +61,7 @@ static const CAmount DEFAULT_TRANSACTION_FEE = 0; //! -paytxfee will warn if called with a higher fee than this amount (in satoshis) per KB static const CAmount nHighTransactionFeeWarning = 0.01 * COIN; //! -fallbackfee default -static const CAmount DEFAULT_FALLBACK_FEE = 20000; +static const CAmount DEFAULT_FALLBACK_FEE = 1000; //! -mintxfee default static const CAmount DEFAULT_TRANSACTION_MINFEE = 1000; //! -maxtxfee default From b9c2a3934484b31d533a55f4086835d990308e94 Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Tue, 12 Feb 2019 13:03:40 -0500 Subject: [PATCH 06/46] Fix potential deadlock (#132) * add missing locks to coins_test * relock in FetchCoin * re-enable startupshutdowntest destructor * update cpp test suite to use regtest * adjust shutdown of test suite * make CChain tip atomic, it no longer needs cs_main * add nodestate manager and accessor. mapnodestate no longer needs cs_main * Misbehaving no longer requires cs_main * remove lingering cs_main locks that are no longer needed * check for valid tip before trying to use it * give mapBlockIndex its own lock * initalize seed array * replace BlockConnected, Disconnected, and Checked with SyncWithWallets * pass block pointers instead of shared pointers * formatting fix * lock cs_main before cs_mapBlockIndex when loading the block index * add read and write locks for shared mutexs * fix issue where we try to recursively grab readlock on mempool.cs --- src/.formatted-files | 2 + src/Makefile.am | 4 +- src/base58.h | 3 +- src/blockgeneration/miner.cpp | 24 ++- src/blockgeneration/minter.cpp | 26 +-- src/chain/chain.cpp | 4 +- src/chain/chain.h | 30 ++- src/chain/chainman.cpp | 52 +++-- src/chain/chainman.h | 7 +- src/chain/checkpoints.cpp | 10 +- src/chain/tx.cpp | 2 +- src/coins.cpp | 2 +- src/eccoind.cpp | 1 + src/init.cpp | 12 +- src/kernel.cpp | 11 +- src/key.cpp | 2 +- src/main.cpp | 45 ++-- src/main.h | 18 -- src/net/messages.cpp | 366 ++++++++++---------------------- src/net/messages.h | 94 +-------- src/net/nodestate.cpp | 24 +++ src/net/nodestate.h | 143 +++++++++++++ src/processblock.cpp | 238 +++------------------ src/processblock.h | 28 ++- src/processheader.cpp | 20 +- src/rest.cpp | 11 +- src/rpc/rpcblockchain.cpp | 94 ++++++--- src/rpc/rpcmining.cpp | 21 +- src/rpc/rpcrawtransaction.cpp | 20 +- src/rpc/rpcwallet.cpp | 5 +- src/sync.cpp | 250 ++++++++++++++++++---- src/sync.h | 373 +++++++++++++++++++++++++++++---- src/test/coins_tests.cpp | 14 +- src/test/miner_tests.cpp | 2 +- src/test/test_bitcoin.cpp | 19 +- src/txmempool.cpp | 55 +++-- src/txmempool.h | 14 +- src/util/util.cpp | 2 +- src/util/util.h | 6 + src/validationinterface.cpp | 19 +- src/validationinterface.h | 27 +-- src/wallet/wallet.cpp | 121 ++++------- src/wallet/wallet.h | 7 +- 43 files changed, 1234 insertions(+), 994 deletions(-) create mode 100644 src/net/nodestate.cpp create mode 100644 src/net/nodestate.h diff --git a/src/.formatted-files b/src/.formatted-files index ceb63e99..1071b7cb 100644 --- a/src/.formatted-files +++ b/src/.formatted-files @@ -91,6 +91,8 @@ net/netaddress.cpp net/netaddress.h net/netbase.cpp net/netbase.h +net/nodestate.cpp +net/nodestate.h net/protocol.cpp net/protocol.h networks/netman.cpp diff --git a/src/Makefile.am b/src/Makefile.am index a9ef0f30..0be5a3c6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -87,6 +87,7 @@ BITCOIN_CORE_H = \ net/net.h \ net/netaddress.h \ net/netbase.h \ + net/nodestate.h \ net/protocol.h \ networks/netman.h \ networks/network.h \ @@ -159,6 +160,7 @@ libbitcoin_server_a-clientversion.$(OBJEXT): build/build.h libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIC_FLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ + sync.cpp \ net/addrdb.cpp \ net/addrman.cpp \ blockgeneration/blockgeneration.cpp \ @@ -214,6 +216,7 @@ libbitcoin_server_a_SOURCES = \ processheader.cpp \ processtx.cpp \ chain/tx.cpp \ + net/nodestate.cpp \ net/protocol.cpp \ pubkey.cpp \ crypto/pbkdf2.cpp \ @@ -235,7 +238,6 @@ libbitcoin_server_a_SOURCES = \ random.cpp \ rpc/rpcprotocol.cpp \ support/cleanse.cpp \ - sync.cpp \ uint256.cpp \ util/util.cpp \ util/utilmoneystr.cpp \ diff --git a/src/base58.h b/src/base58.h index c15c88e4..0d9aab22 100644 --- a/src/base58.h +++ b/src/base58.h @@ -29,7 +29,6 @@ #ifndef BITCOIN_BASE58_H #define BITCOIN_BASE58_H -#include "init.h" #include "key.h" #include "networks/netman.h" #include "networks/networktemplate.h" @@ -41,6 +40,8 @@ #include #include +extern CNetworkManager *pnetMan; + /** * Encode a byte sequence as a base58-encoded string. * pbegin and pend cannot be NULL, unless both are. diff --git a/src/blockgeneration/miner.cpp b/src/blockgeneration/miner.cpp index e7d17375..891a267b 100644 --- a/src/blockgeneration/miner.cpp +++ b/src/blockgeneration/miner.cpp @@ -161,7 +161,8 @@ std::unique_ptr CreateNewPoWBlock(CWallet *pwallet, const CScrip pblock->nBits = GetNextTargetRequired(pindexPrev, false); // Collect memory pool transactions into the block { - LOCK2(cs_main, mempool.cs); + LOCK(cs_main); + READLOCK(mempool.cs); CBlockIndex *_pindexPrev = pnetMan->getChainActive()->chainActive.Tip(); const int nHeight = _pindexPrev->nHeight + 1; pblock->nTime = GetAdjustedTime(); @@ -181,7 +182,7 @@ std::unique_ptr CreateNewPoWBlock(CWallet *pwallet, const CScrip { double dPriority = mi->GetPriority(nHeight); CAmount dummy; - mempool.ApplyDeltas(mi->GetTx().GetHash(), dPriority, dummy); + mempool._ApplyDeltas(mi->GetTx().GetHash(), dPriority, dummy); vecPriority.push_back(TxCoinAgePriority(dPriority, mi)); } std::make_heap(vecPriority.begin(), vecPriority.end(), pricomparer); @@ -369,9 +370,7 @@ void FormatHashBuffers(CBlock *pblock, char *pmidstate, char *pdata, char *phash } -bool CheckWork(const std::shared_ptr pblock, - CWallet &wallet, - boost::shared_ptr coinbaseScript) +bool CheckWork(const CBlock *pblock, CWallet &wallet, boost::shared_ptr coinbaseScript) { arith_uint256 hash = UintToArith256(pblock->GetHash()); arith_uint256 hashTarget = arith_uint256(pblock->nBits); @@ -386,16 +385,22 @@ bool CheckWork(const std::shared_ptr pblock, // Found a solution { - LOCK(cs_main); - if (pblock->hashPrevBlock != pnetMan->getChainActive()->chainActive.Tip()->GetBlockHash()) + CBlockIndex *ptip = pnetMan->getChainActive()->chainActive.Tip(); + if (ptip == nullptr) + { + return false; + } + if (pblock->hashPrevBlock != ptip->GetBlockHash()) + { return error("BMiner : generated block is stale"); + } // Remove key from key pool coinbaseScript->KeepScript(); // Track how many getdata requests this block gets { - LOCK(wallet.cs_wallet); + LOCK2(cs_main, wallet.cs_wallet); wallet.mapRequestCount[pblock->GetHash()] = 0; } @@ -504,8 +509,7 @@ void EccMiner(CWallet *pwallet) break; } SetThreadPriority(THREAD_PRIORITY_NORMAL); - const std::shared_ptr spblock = std::make_shared(*pblock); - CheckWork(spblock, *pwalletMain, coinbaseScript); + CheckWork(pblock, *pwalletMain, coinbaseScript); SetThreadPriority(THREAD_PRIORITY_LOWEST); break; } diff --git a/src/blockgeneration/minter.cpp b/src/blockgeneration/minter.cpp index 07ec50c3..2536f58f 100644 --- a/src/blockgeneration/minter.cpp +++ b/src/blockgeneration/minter.cpp @@ -32,9 +32,7 @@ extern CWallet *pwalletMain; int64_t nLastCoinStakeSearchInterval = 0; -bool CheckStake(const std::shared_ptr pblock, - CWallet &wallet, - boost::shared_ptr coinbaseScript) +bool CheckStake(const CBlock *pblock, CWallet &wallet, boost::shared_ptr coinbaseScript) { //// debug print LogPrintf("Minter:\n"); @@ -50,16 +48,20 @@ bool CheckStake(const std::shared_ptr pblock, // Found a solution { - LOCK(cs_main); - if (pblock->hashPrevBlock != pnetMan->getChainActive()->chainActive.Tip()->GetBlockHash()) + CBlockIndex *ptip = pnetMan->getChainActive()->chainActive.Tip(); + if (ptip == nullptr) + { + return false; + } + if (pblock->hashPrevBlock != ptip->GetBlockHash()) { - return error("Minter : generated block is stale"); + return error("BMiner : generated block is stale"); } // Remove key from key pool coinbaseScript->KeepScript(); // Track how many getdata requests this block gets { - LOCK(wallet.cs_wallet); + LOCK2(cs_main, wallet.cs_wallet); wallet.mapRequestCount[pblock->GetHash()] = 0; } // Process this block the same as if we had received it from another node @@ -147,14 +149,15 @@ std::unique_ptr CreateNewPoSBlock(CWallet *pwallet, const CScrip nLastCoinStakeSearchInterval = nSearchTime - nLastCoinStakeSearchTime; nLastCoinStakeSearchTime = nSearchTime; } - MilliSleep(100); + MilliSleep(50); if (shutdown_threads.load()) return nullptr; } // Collect memory pool transactions into the block { - LOCK2(cs_main, mempool.cs); + LOCK(cs_main); + READLOCK(mempool.cs); CBlockIndex *_pindexPrev = pnetMan->getChainActive()->chainActive.Tip(); const int nHeight = _pindexPrev->nHeight + 1; pblock->nTime = GetAdjustedTime(); @@ -174,7 +177,7 @@ std::unique_ptr CreateNewPoSBlock(CWallet *pwallet, const CScrip { double dPriority = mi->GetPriority(nHeight); CAmount dummy; - mempool.ApplyDeltas(mi->GetTx().GetHash(), dPriority, dummy); + mempool._ApplyDeltas(mi->GetTx().GetHash(), dPriority, dummy); vecPriority.push_back(TxCoinAgePriority(dPriority, mi)); } std::make_heap(vecPriority.begin(), vecPriority.end(), pricomparer); @@ -372,8 +375,7 @@ void EccMinter(CWallet *pwallet) } LogPrintf("CPUMiner : proof-of-stake block found %s\n", pblock->GetHash().ToString().c_str()); SetThreadPriority(THREAD_PRIORITY_NORMAL); - const std::shared_ptr spblock = std::make_shared(*pblock); - CheckStake(spblock, *pwalletMain, coinbaseScript); + CheckStake(pblock, *pwalletMain, coinbaseScript); SetThreadPriority(THREAD_PRIORITY_LOWEST); MilliSleep(1000); // 1 second delay continue; diff --git a/src/chain/chain.cpp b/src/chain/chain.cpp index f255aef9..b2e8bdf4 100644 --- a/src/chain/chain.cpp +++ b/src/chain/chain.cpp @@ -25,12 +25,14 @@ */ void CChain::SetTip(CBlockIndex *pindex) { - if (pindex == NULL) + if (pindex == nullptr) { vChain.clear(); + tip = nullptr; return; } vChain.resize(pindex->nHeight + 1); + tip = pindex; while (pindex && vChain[pindex->nHeight] != pindex) { vChain[pindex->nHeight] = pindex; diff --git a/src/chain/chain.h b/src/chain/chain.h index d077ce49..b08ff61f 100644 --- a/src/chain/chain.h +++ b/src/chain/chain.h @@ -28,6 +28,7 @@ #include "tinyformat.h" #include "uint256.h" +#include #include /** An in-memory indexed chain of blocks. */ @@ -36,16 +37,23 @@ class CChain private: std::vector vChain; + std::atomic tip; + public: - ~CChain() { vChain.clear(); } + CChain() : tip(nullptr) {} + ~CChain() + { + vChain.clear(); + tip = nullptr; + } /** Returns the index entry for the genesis block of this chain, or NULL if none. */ - CBlockIndex *Genesis() const { return vChain.size() > 0 ? vChain[0] : NULL; } + CBlockIndex *Genesis() const { return vChain.size() > 0 ? vChain[0] : nullptr; } /** Returns the index entry for the tip of this chain, or NULL if none. */ - CBlockIndex *Tip() const { return vChain.size() > 0 ? vChain[vChain.size() - 1] : NULL; } + CBlockIndex *Tip() const { return tip; } CBlockIndex *AtHeight(int nHeight) const { if (nHeight < 0 || nHeight >= (int)vChain.size()) - return NULL; + return nullptr; return vChain[nHeight]; } @@ -53,7 +61,7 @@ class CChain CBlockIndex *operator[](int nHeight) const { if (nHeight < 0 || nHeight >= (int)vChain.size()) - return NULL; + return nullptr; return vChain[nHeight]; } @@ -63,6 +71,12 @@ class CChain return a.vChain.size() == b.vChain.size() && a.vChain[a.vChain.size() - 1] == b.vChain[b.vChain.size() - 1]; } + void operator=(const CChain &a) + { + vChain = a.vChain; + tip.store(a.tip.load()); + } + /** Efficiently check whether a block is present in this chain. */ bool Contains(const CBlockIndex *pindex) const { return (*this)[pindex->nHeight] == pindex; } /** Find the successor of a block in this chain, or NULL if the given index is not found or is the tip. */ @@ -71,16 +85,16 @@ class CChain if (Contains(pindex)) return (*this)[pindex->nHeight + 1]; else - return NULL; + return nullptr; } /** Return the maximal height in the chain. Is equal to chain.Tip() ? chain.Tip()->nHeight : -1. */ - int Height() const { return vChain.size() - 1; } + int Height() const { return tip.load() ? tip.load()->nHeight : -1; } /** Set/initialize a chain with a given tip. */ void SetTip(CBlockIndex *pindex); /** Return a CBlockLocator that refers to a block in this chain (by default the tip). */ - CBlockLocator GetLocator(const CBlockIndex *pindex = NULL) const; + CBlockLocator GetLocator(const CBlockIndex *pindex = nullptr) const; /** Find the last common block between this chain and a block index entry. */ const CBlockIndex *FindFork(const CBlockIndex *pindex) const; diff --git a/src/chain/chainman.cpp b/src/chain/chainman.cpp index 95e05300..c098ab1e 100644 --- a/src/chain/chainman.cpp +++ b/src/chain/chainman.cpp @@ -23,14 +23,26 @@ #include "kernel.h" #include "main.h" #include "net/messages.h" +#include "net/nodestate.h" #include "networks/netman.h" #include "processblock.h" #include "processheader.h" #include "txmempool.h" #include "undo.h" +CBlockIndex *CChainManager::LookupBlockIndex(const uint256 &hash) +{ + READLOCK(cs_mapBlockIndex); + BlockMap::iterator mi = mapBlockIndex.find(hash); + if (mi == mapBlockIndex.end()) + return nullptr; + return mi->second; // I can return this CBlockIndex because header pointers are never deleted +} + + CBlockIndex *CChainManager::AddToBlockIndex(const CBlockHeader &block) { + WRITELOCK(cs_mapBlockIndex); // Check for duplicate uint256 hash = block.GetHash(); BlockMap::iterator it = mapBlockIndex.find(hash); @@ -65,6 +77,7 @@ CBlockIndex *CChainManager::AddToBlockIndex(const CBlockHeader &block) CBlockIndex *CChainManager::FindForkInGlobalIndex(const CChain &chain, const CBlockLocator &locator) { + READLOCK(cs_mapBlockIndex); // Find the first block the caller has in the main chain for (auto const &hash : locator.vHave) { @@ -81,8 +94,8 @@ CBlockIndex *CChainManager::FindForkInGlobalIndex(const CChain &chain, const CBl bool CChainManager::IsInitialBlockDownload() { + READLOCK(cs_mapBlockIndex); const CNetworkTemplate &chainParams = pnetMan->getActivePaymentNetwork(); - LOCK(cs_main); if (fImporting || fReindex) return true; if (fCheckpointsEnabled && chainActive.Height() < Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints())) @@ -101,6 +114,7 @@ CBlockIndex *CChainManager::InsertBlockIndex(uint256 hash) { if (hash.IsNull()) return NULL; + WRITELOCK(cs_mapBlockIndex); // Return existing BlockMap::iterator mi = mapBlockIndex.find(hash); @@ -135,9 +149,7 @@ bool CChainManager::InitBlockIndex(const CNetworkTemplate &chainparams) { try { - std::shared_ptr spblock = std::make_shared(); - CBlock &block = *spblock; - block = chainparams.GenesisBlock(); + CBlock block = chainparams.GenesisBlock(); // Start new block file unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); CDiskBlockPos blockPos; @@ -162,7 +174,7 @@ bool CChainManager::InitBlockIndex(const CNetworkTemplate &chainparams) pindex->SetStakeModifier(nStakeModifier); if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) return error("InitBlockIndex(): genesis block not accepted"); - if (!ActivateBestChain(state, chainparams, spblock)) + if (!ActivateBestChain(state, chainparams, &block)) return error("InitBlockIndex(): genesis block cannot be activated"); // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); @@ -192,6 +204,9 @@ bool CChainManager::LoadBlockIndexDB() return false; } + LOCK(cs_main); + WRITELOCK(cs_mapBlockIndex); + LogPrintf("LoadBlockIndexGuts %15dms\n", GetTimeMillis() - nStart); if (shutdown_threads.load()) @@ -318,6 +333,7 @@ bool CChainManager::LoadBlockIndexDB() bool CChainManager::LoadExternalBlockFile(const CNetworkTemplate &chainparams, FILE *fileIn, CDiskBlockPos *dbp) { + WRITELOCK(cs_mapBlockIndex); // std::map of disk positions for blocks with unknown parent (only used for reindex) static std::multimap mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); @@ -366,8 +382,7 @@ bool CChainManager::LoadExternalBlockFile(const CNetworkTemplate &chainparams, F dbp->nPos = nBlockPos; blkdat.SetLimit(nBlockPos + nSize); blkdat.SetPos(nBlockPos); - std::shared_ptr spblock = std::make_shared(); - CBlock &block = *spblock; + CBlock block; blkdat >> block; nRewind = blkdat.GetPos(); @@ -387,7 +402,7 @@ bool CChainManager::LoadExternalBlockFile(const CNetworkTemplate &chainparams, F if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { CValidationState state; - if (ProcessNewBlock(state, chainparams, NULL, spblock, true, dbp)) + if (ProcessNewBlock(state, chainparams, NULL, &block, true, dbp)) nLoaded++; if (state.IsError()) break; @@ -416,7 +431,7 @@ bool CChainManager::LoadExternalBlockFile(const CNetworkTemplate &chainparams, F LogPrintf("%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), head.ToString()); CValidationState dummy; - if (ProcessNewBlock(dummy, chainparams, NULL, spblock, true, &it->second)) + if (ProcessNewBlock(dummy, chainparams, NULL, &block, true, &it->second)) { nLoaded++; queue.push_back(block.GetHash()); @@ -446,9 +461,9 @@ void CChainManager::UnloadBlockIndex() { LOCK(cs_main); setBlockIndexCandidates.clear(); - chainActive.SetTip(NULL); - pindexBestInvalid = NULL; - pindexBestHeader = NULL; + chainActive.SetTip(nullptr); + pindexBestInvalid = nullptr; + pindexBestHeader = nullptr; mempool.clear(); mapOrphanTransactions.clear(); mapOrphanTransactionsByPrev.clear(); @@ -462,17 +477,20 @@ void CChainManager::UnloadBlockIndex() nPreferredDownload = 0; setDirtyBlockIndex.clear(); setDirtyFileInfo.clear(); - mapNodeState.clear(); - recentRejects.reset(NULL); + nodestateman.Clear(); + recentRejects.reset(nullptr); versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { warningcache[b].clear(); } - for (auto &entry : mapBlockIndex) { - delete entry.second; + WRITELOCK(cs_mapBlockIndex); + for (auto &entry : mapBlockIndex) + { + delete entry.second; + } + mapBlockIndex.clear(); } - mapBlockIndex.clear(); } diff --git a/src/chain/chainman.h b/src/chain/chainman.h index b308a0b8..5637b160 100644 --- a/src/chain/chainman.h +++ b/src/chain/chainman.h @@ -37,8 +37,10 @@ typedef std::unordered_map BlockMap; class CChainManager { public: + CSharedCriticalSection cs_mapBlockIndex; + /** map containing all block indexs ever seen for this chain */ - BlockMap mapBlockIndex; + BlockMap mapBlockIndex GUARDED_BY(cs_mapBlockIndex); /** The currently-connected chain of blocks (protected by cs_main). */ CChain chainActive; @@ -79,6 +81,7 @@ class CChainManager void operator=(const CChainManager &oldMan) { + WRITELOCK(cs_mapBlockIndex); mapBlockIndex = oldMan.mapBlockIndex; chainActive = oldMan.chainActive; pindexBestHeader = oldMan.pindexBestHeader; @@ -86,6 +89,8 @@ class CChainManager pblocktree.reset(oldMan.pblocktree.get()); } + /** Look up the block index entry for a given block hash. returns nullptr if it does not exist */ + CBlockIndex *LookupBlockIndex(const uint256 &hash); /** Add a new block index entry for a given block recieved from the network */ CBlockIndex *AddToBlockIndex(const CBlockHeader &block); diff --git a/src/chain/checkpoints.cpp b/src/chain/checkpoints.cpp index 9c0115a3..50035ae6 100644 --- a/src/chain/checkpoints.cpp +++ b/src/chain/checkpoints.cpp @@ -90,11 +90,13 @@ CBlockIndex *GetLastCheckpoint(const CCheckpointData &data) BOOST_REVERSE_FOREACH (const MapCheckpoints::value_type &i, checkpoints) { const uint256 &hash = i.second; - BlockMap::const_iterator t = pnetMan->getChainActive()->mapBlockIndex.find(hash); - if (t != pnetMan->getChainActive()->mapBlockIndex.end()) - return t->second; + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (pindex) + { + return pindex; + } } - return NULL; + return nullptr; } } // namespace Checkpoints diff --git a/src/chain/tx.cpp b/src/chain/tx.cpp index 44f395e4..1aa41959 100644 --- a/src/chain/tx.cpp +++ b/src/chain/tx.cpp @@ -345,7 +345,7 @@ bool GetTransaction(const uint256 &hash, uint256 &hashBlock, bool fAllowSlow) { - CBlockIndex *pindexSlow = NULL; + CBlockIndex *pindexSlow = nullptr; LOCK(cs_main); diff --git a/src/coins.cpp b/src/coins.cpp index b68f4f64..c353e83e 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -91,7 +91,7 @@ size_t CCoinsViewCache::ResetCachedCoinUsage() const CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { - AssertLockHeld(cs_utxo); + LOCK(cs_utxo); CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) return it; diff --git a/src/eccoind.cpp b/src/eccoind.cpp index 67efdd03..58f824e5 100644 --- a/src/eccoind.cpp +++ b/src/eccoind.cpp @@ -25,6 +25,7 @@ #include "networks/netman.h" #include "noui.h" #include "rpc/rpcserver.h" +#include "sync.h" #include "threadgroup.h" #include "util/util.h" diff --git a/src/init.cpp b/src/init.cpp index 5745369c..d131f7bb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1434,6 +1434,7 @@ bool AppInit2(thread_group &threadGroup) LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024)); + LOCK(cs_main); bool fLoaded = false; while (!fLoaded) { @@ -1478,10 +1479,14 @@ bool AppInit2(thread_group &threadGroup) } // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). - if (!pnetMan->getChainActive()->mapBlockIndex.empty() && - pnetMan->getChainActive()->mapBlockIndex.count(chainparams.GetConsensus().hashGenesisBlock) == 0) { - return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); + if (!pnetMan->getChainActive()->mapBlockIndex.empty() && + pnetMan->getChainActive()->mapBlockIndex.count(chainparams.GetConsensus().hashGenesisBlock) == + 0) + { + return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); + } } // Initialize the block index (no-op if non-empty database was already loaded) @@ -1510,7 +1515,6 @@ bool AppInit2(thread_group &threadGroup) LogPrintf("Verifying blocks..."); { - LOCK(cs_main); CBlockIndex *tip = pnetMan->getChainActive()->chainActive.Tip(); if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { diff --git a/src/kernel.cpp b/src/kernel.cpp index 4281bb80..6f13a674 100644 --- a/src/kernel.cpp +++ b/src/kernel.cpp @@ -39,10 +39,11 @@ static bool GetKernelStakeModifier(uint256 hashBlockFrom, uint256 &nStakeModifier) { nStakeModifier.SetNull(); - if (!pnetMan->getChainActive()->mapBlockIndex.count(hashBlockFrom)) + const CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hashBlockFrom); + if (!pindex) + { return error("GetKernelStakeModifier() : block not indexed"); - const CBlockIndex *pindex = pnetMan->getChainActive()->mapBlockIndex[hashBlockFrom]; - + } int blocksToGo = 5; if (pnetMan->getChainActive()->chainActive.Tip()->nHeight >= 1504350) { @@ -128,7 +129,7 @@ bool ComputeNextStakeModifier(const CBlockIndex *pindexPrev, const CTransaction // Read block header CBlock block; - CBlockIndex *index = pnetMan->getChainActive()->mapBlockIndex[blockHashOfTx]; + CBlockIndex *index = pnetMan->getChainActive()->LookupBlockIndex(blockHashOfTx); if (!ReadBlockFromDisk(block, index, pnetMan->getActivePaymentNetwork()->GetConsensus())) // unable to read block of previous transaction @@ -305,7 +306,7 @@ bool CheckProofOfStake(int nHeight, const CTransaction &tx, uint256 &hashProofOf // Read block header CBlock block; - CBlockIndex *index = pnetMan->getChainActive()->mapBlockIndex[blockHashOfTx]; + CBlockIndex *index = pnetMan->getChainActive()->LookupBlockIndex(blockHashOfTx); if (!ReadBlockFromDisk(block, index, pnetMan->getActivePaymentNetwork()->GetConsensus())) // unable to read block of previous transaction diff --git a/src/key.cpp b/src/key.cpp index 53556f51..31d601ff 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -442,7 +442,7 @@ void ECC_Start() { // Pass in a random blinding seed to the secp256k1 context. - uint8_t seed[32]; + uint8_t seed[32] = {0}; LockObject(seed); GetRandBytes(seed, 32); bool ret = secp256k1_context_randomize(ctx, seed); diff --git a/src/main.cpp b/src/main.cpp index 8a5e2fca..2e7424d3 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -169,12 +169,7 @@ int nPeersWithValidatedDownloads = 0; // Registration of network node signals. // -int GetHeight() -{ - LOCK(cs_main); - return pnetMan->getChainActive()->chainActive.Height(); -} - +int GetHeight() { return pnetMan->getChainActive()->chainActive.Height(); } ////////////////////////////////////////////////////////////////////////////// // // mapOrphanTransactions @@ -530,7 +525,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, // Check for conflicts with in-memory transactions { - LOCK(pool.cs); // protect pool.mapNextTx + READLOCK(pool.cs); // protect pool.mapNextTx for (auto const &txin : tx.vin) { auto itConflicting = pool.mapNextTx.find(txin.prevout); @@ -549,7 +544,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, CAmount nValueIn = 0; LockPoints lp; { - LOCK(pool.cs); + WRITELOCK(pool.cs); CCoinsViewMemPool viewMemPool(pnetMan->getChainActive()->pcoinsTip.get(), pool); view.SetBackend(viewMemPool); @@ -789,7 +784,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, __func__, hash.ToString(), FormatStateMessage(state)); } { - LOCK(pool.cs); + WRITELOCK(pool.cs); if (!pool._CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { @@ -798,7 +793,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, } { - LOCK(pool.cs); + WRITELOCK(pool.cs); // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !pnetMan->getChainActive()->IsInitialBlockDownload()); } @@ -813,8 +808,13 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool full"); } } + + if (!fRejectAbsurdFee) + { + SyncWithWallets(ptx, nullptr, -1); + } } - GetMainSignals().TransactionAddedToMempool(ptx); + return true; } @@ -895,6 +895,10 @@ bool ReadBlockFromDisk(CBlock &block, const CDiskBlockPos &pos, const Consensus: bool ReadBlockFromDisk(CBlock &block, const CBlockIndex *pindex, const Consensus::Params &consensusParams) { + if (!pindex) + { + return false; + } if (!ReadBlockFromDisk(block, pindex->GetBlockPos(), consensusParams)) { return false; @@ -942,8 +946,7 @@ bool CScriptCheck::operator()() int GetSpendHeight(const CCoinsViewCache &inputs) { - LOCK(cs_main); - CBlockIndex *pindexPrev = pnetMan->getChainActive()->mapBlockIndex.find(inputs.GetBestBlock())->second; + CBlockIndex *pindexPrev = pnetMan->getChainActive()->LookupBlockIndex(inputs.GetBestBlock()); return pindexPrev->nHeight + 1; } @@ -1325,15 +1328,18 @@ bool InvalidateBlock(CValidationState &state, const Consensus::Params &consensus // The resulting new best tip may not be in setBlockIndexCandidates anymore, so // add it again. - BlockMap::iterator it = pnetMan->getChainActive()->mapBlockIndex.begin(); - while (it != pnetMan->getChainActive()->mapBlockIndex.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && - !setBlockIndexCandidates.value_comp()(it->second, pnetMan->getChainActive()->chainActive.Tip())) + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); + BlockMap::iterator it = pnetMan->getChainActive()->mapBlockIndex.begin(); + while (it != pnetMan->getChainActive()->mapBlockIndex.end()) { - setBlockIndexCandidates.insert(it->second); + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && + !setBlockIndexCandidates.value_comp()(it->second, pnetMan->getChainActive()->chainActive.Tip())) + { + setBlockIndexCandidates.insert(it->second); + } + it++; } - it++; } InvalidChainFound(pindex); @@ -1348,6 +1354,7 @@ bool ReconsiderBlock(CValidationState &state, CBlockIndex *pindex) int nHeight = pindex->nHeight; + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); // Remove the invalidity flag from this block if (!pindex->IsValid()) { diff --git a/src/main.h b/src/main.h index 9812f823..98ccce61 100644 --- a/src/main.h +++ b/src/main.h @@ -239,24 +239,6 @@ unsigned int GetNextTargetRequired(const CBlockIndex *pindexLast, bool fProofOfS int64_t GetProofOfWorkReward(int64_t nFees, const int nHeight, uint256 prevHash); int64_t GetProofOfStakeReward(int64_t nCoinAge, int nHeight); -/** - * Process an incoming block. This only returns after the best known valid - * block is made active. Note that it does not, however, guarantee that the - * specific block passed to it has been checked for validity! - * - * @param[out] state This may be set to an Error state if any error occurred processing it, including during - * validation/connection/etc of otherwise unrelated blocks during reorganisation; or it may be set to an Invalid state - * if pblock is itself invalid (but this is not guaranteed even when the block is checked). If you want to *possibly* - * get feedback on whether pblock is valid, you must also install a CValidationInterface (see validationinterface.h) - - * this will have its BlockChecked method called whenever *any* block completes validation. - * @param[in] pfrom The node which we are receiving the block from; it is added to mapBlockSource and may be - * penalised if the block is invalid. - * @param[in] pblock The block we want to process. - * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and - * whitelisted peers. - * @param[out] dbp If pblock is stored to disk (or already there), this will be set to its location. - * @return True if state.IsValid() - */ /** Check whether enough disk space is available for an incoming block */ bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); /** Open a block file (blk?????.dat) */ diff --git a/src/net/messages.cpp b/src/net/messages.cpp index 5ec03ebc..12bb745b 100644 --- a/src/net/messages.cpp +++ b/src/net/messages.cpp @@ -28,6 +28,7 @@ #include "main.h" #include "merkleblock.h" #include "net/addrman.h" +#include "net/nodestate.h" #include "net/protocol.h" #include "networks/netman.h" #include "networks/networktemplate.h" @@ -86,29 +87,10 @@ CCriticalSection cs_mapRelay; std::map mapRelay; std::deque::iterator> > vRelayExpiration; -static CCriticalSection cs_most_recent_block; -static std::shared_ptr most_recent_block; -static uint256 most_recent_block_hash; - -/** Map maintaining per-node state. Requires cs_main. */ -std::map mapNodeState; - uint64_t nLocalHostNonce = 0; extern CCriticalSection cs_mapInboundConnectionTracker; extern std::map mapInboundConnectionTracker; -// Requires cs_main. -CNodeState *State(NodeId pnode) -{ - std::map::iterator it = mapNodeState.find(pnode); - if (it == mapNodeState.end()) - { - return nullptr; - } - - return &it->second; -} - uint32_t GetFetchFlags(CNode *pfrom, const CBlockIndex *pprev, const Consensus::Params &chainparams) { uint32_t nFetchFlags = 0; @@ -145,14 +127,7 @@ void PushNodeVersion(CNode *pnode, CConnman &connman, int64_t nTime) void InitializeNode(CNode *pnode, CConnman &connman) { - CAddress addr = pnode->addr; - std::string addrName = pnode->GetAddrName(); - NodeId nodeid = pnode->GetId(); - { - LOCK(cs_main); - mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), - std::forward_as_tuple(addr, std::move(addrName))); - } + nodestateman.InitializeNodeState(pnode); if (!pnode->fInbound) { @@ -163,8 +138,7 @@ void InitializeNode(CNode *pnode, CConnman &connman) void FinalizeNode(NodeId nodeid, bool &fUpdateConnectionTime) { fUpdateConnectionTime = false; - LOCK(cs_main); - CNodeState *state = State(nodeid); + CNodeStateAccessor state(nodestateman, nodeid); if (state->fSyncStarted) { @@ -176,6 +150,7 @@ void FinalizeNode(NodeId nodeid, bool &fUpdateConnectionTime) fUpdateConnectionTime = true; } + LOCK(cs_main); for (const QueuedBlock &entry : state->vBlocksInFlight) { mapBlocksInFlight.erase(entry.hash); @@ -205,9 +180,9 @@ void FinalizeNode(NodeId nodeid, bool &fUpdateConnectionTime) nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); assert(nPeersWithValidatedDownloads >= 0); - mapNodeState.erase(nodeid); + nodestateman.RemoveNodeState(nodeid); - if (mapNodeState.empty()) + if (nodestateman.Empty()) { // Do a consistency check after the last peer is removed. assert(mapBlocksInFlight.empty()); @@ -216,17 +191,18 @@ void FinalizeNode(NodeId nodeid, bool &fUpdateConnectionTime) } } +// TODO might require cs_main bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { - LOCK(cs_main); - CNodeState *state = State(nodeid); - if (state == nullptr) + CNodeStateAccessor state(nodestateman, nodeid); + if (state.IsNull()) { return false; } stats.nMisbehavior = state->nMisbehavior; stats.nSyncHeight = state->pindexBestKnownBlock ? state->pindexBestKnownBlock->nHeight : -1; stats.nCommonHeight = state->pindexLastCommonBlock ? state->pindexLastCommonBlock->nHeight : -1; + for (const QueuedBlock &queue : state->vBlocksInFlight) { if (queue.pindex) @@ -237,7 +213,6 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) return true; } -// Requires cs_main. void Misbehaving(NodeId pnode, int howmuch, const std::string &reason) { if (howmuch == 0) @@ -245,8 +220,8 @@ void Misbehaving(NodeId pnode, int howmuch, const std::string &reason) return; } - CNodeState *state = State(pnode); - if (state == nullptr) + CNodeStateAccessor state(nodestateman, pnode); + if (state.IsNull()) { return; } @@ -281,7 +256,7 @@ bool MarkBlockAsReceived(const uint256 &hash) mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end()) { - CNodeState *state = State(itInFlight->second.first); + CNodeStateAccessor state(nodestateman, itInFlight->second.first); state->nBlocksInFlightValidHeaders -= itInFlight->second.second->fValidatedHeaders; if (state->nBlocksInFlightValidHeaders == 0 && itInFlight->second.second->fValidatedHeaders) { @@ -295,7 +270,6 @@ bool MarkBlockAsReceived(const uint256 &hash) } state->vBlocksInFlight.erase(itInFlight->second.second); state->nBlocksInFlight--; - state->nStallingSince = 0; mapBlocksInFlight.erase(itInFlight); return true; } @@ -484,10 +458,10 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRE void MarkBlockAsInFlight(NodeId nodeid, const uint256 &hash, const Consensus::Params &consensusParams, - const CBlockIndex *pindex = NULL) + const CBlockIndex *pindex = nullptr) { - CNodeState *state = State(nodeid); - assert(state != NULL); + CNodeStateAccessor state(nodestateman, nodeid); + assert(state.IsNull() == false); // Make sure it's not listed somewhere already. MarkBlockAsReceived(hash); @@ -511,24 +485,26 @@ void MarkBlockAsInFlight(NodeId nodeid, /** Check whether the last unknown block a peer advertized is not yet known. */ void ProcessBlockAvailability(NodeId nodeid) { - CNodeState *state = State(nodeid); - assert(state != NULL); + CNodeStateAccessor state(nodestateman, nodeid); + assert(state.IsNull() == false); if (!state->hashLastUnknownBlock.IsNull()) { - BlockMap::iterator itOld = pnetMan->getChainActive()->mapBlockIndex.find(state->hashLastUnknownBlock); - if (itOld != pnetMan->getChainActive()->mapBlockIndex.end() && itOld->second->nChainWork > 0) + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(state->hashLastUnknownBlock); + if (pindex && pindex->nChainWork > 0) { - if (state->pindexBestKnownBlock == NULL || - itOld->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) - state->pindexBestKnownBlock = itOld->second; + if (state->pindexBestKnownBlock == NULL || pindex->nChainWork >= state->pindexBestKnownBlock->nChainWork) + { + state->pindexBestKnownBlock = pindex; + } state->hashLastUnknownBlock.SetNull(); } } } -void UpdatePreferredDownload(CNode *node, CNodeState *state) +void UpdatePreferredDownload(CNode *node) { + CNodeStateAccessor state(nodestateman, node->GetId()); nPreferredDownload -= state->fPreferredDownload; // Whether this node should be marked as a preferred download node. @@ -548,8 +524,8 @@ void FindNextBlocksToDownload(NodeId nodeid, return; vBlocks.reserve(vBlocks.size() + count); - CNodeState *state = State(nodeid); - assert(state != NULL); + CNodeStateAccessor state(nodestateman, nodeid); + assert(state.IsNull() == false); // Make sure pindexBestKnownBlock is up to date, we'll need it. ProcessBlockAvailability(nodeid); @@ -645,29 +621,33 @@ void FindNextBlocksToDownload(NodeId nodeid, } // Requires cs_main -bool PeerHasHeader(CNodeState *state, CBlockIndex *pindex) +bool PeerHasHeader(CNodeState *state, const CBlockIndex *pindex) { if (state->pindexBestKnownBlock && pindex == state->pindexBestKnownBlock->GetAncestor(pindex->nHeight)) + { return true; + } if (state->pindexBestHeaderSent && pindex == state->pindexBestHeaderSent->GetAncestor(pindex->nHeight)) + { return true; + } return false; } /** Update tracking information about which blocks a peer is assumed to have. */ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { - CNodeState *state = State(nodeid); - assert(state != NULL); + CNodeStateAccessor state(nodestateman, nodeid); + assert(state.IsNull() == false); ProcessBlockAvailability(nodeid); - BlockMap::iterator it = pnetMan->getChainActive()->mapBlockIndex.find(hash); - if (it != pnetMan->getChainActive()->mapBlockIndex.end() && it->second->nChainWork > 0) + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (pindex && pindex->nChainWork > 0) { // An actually better block was announced. - if (state->pindexBestKnownBlock == NULL || it->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) - state->pindexBestKnownBlock = it->second; + if (state->pindexBestKnownBlock == NULL || pindex->nChainWork >= state->pindexBestKnownBlock->nChainWork) + state->pindexBestKnownBlock = pindex; } else { @@ -679,7 +659,7 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) static bool SendRejectsAndCheckIfBanned(CNode *pnode, CConnman &connman) { AssertLockHeld(cs_main); - CNodeState &state = *State(pnode->GetId()); + CNodeState state = *CNodeStateAccessor(nodestateman, pnode->GetId()); for (const CBlockReject &reject : state.rejects) { @@ -716,20 +696,6 @@ static bool SendRejectsAndCheckIfBanned(CNode *pnode, CConnman &connman) return false; } -// Requires cs_main -bool PeerHasHeader(CNodeState *state, const CBlockIndex *pindex) -{ - if (state->pindexBestKnownBlock && pindex == state->pindexBestKnownBlock->GetAncestor(pindex->nHeight)) - { - return true; - } - if (state->pindexBestHeaderSent && pindex == state->pindexBestHeaderSent->GetAncestor(pindex->nHeight)) - { - return true; - } - return false; -} - ////////////////////////////////////////////////////////////////////////////// // // blockchain -> download logic notification @@ -741,45 +707,7 @@ PeerLogicValidation::PeerLogicValidation(CConnman *connmanIn) : connman(connmanI recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); } -void PeerLogicValidation::BlockConnected(const std::shared_ptr &pblock, - const CBlockIndex *pindex, - const std::vector &vtxConflicted) -{ - LOCK(cs_main); - - std::vector vOrphanErase; - - for (const CTransactionRef &ptx : pblock->vtx) - { - const CTransaction tx = *ptx; - - // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) - { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout.hash); - if (itByPrev == mapOrphanTransactionsByPrev.end()) - { - continue; - } - for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) - { - const uint256 &orphanHash = *mi; - vOrphanErase.push_back(orphanHash); - } - } - } - - // Erase orphan transactions include or precluded by this block - if (vOrphanErase.size()) - { - for (uint256 &orphanId : vOrphanErase) - { - EraseOrphanTx(orphanId); - } - } -} - -void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr &pblock) +void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const CBlock *pblock) { LOCK(cs_main); static int nHighestFastAnnounce = 0; @@ -789,11 +717,6 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: } nHighestFastAnnounce = pindex->nHeight; uint256 hashBlock(pblock->GetHash()); - { - LOCK(cs_most_recent_block); - most_recent_block_hash = hashBlock; - most_recent_block = pblock; - } connman->ForEachNode([this, pindex, &hashBlock](CNode *pnode) { // TODO: Avoid the repeated-serialization here if (pnode->fDisconnect) @@ -801,7 +724,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: return; } ProcessBlockAvailability(pnode->GetId()); - CNodeState &state = *State(pnode->GetId()); + CNodeState &state = *CNodeStateAccessor(nodestateman, pnode->GetId()); // If the peer has, or we announced to them the previous block already, // but we don't think they have this one, go ahead and announce it. if (!PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) @@ -856,33 +779,6 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, nTimeBestReceived = GetTime(); } -void PeerLogicValidation::BlockChecked(const CBlock &block, const CValidationState &state) -{ - LOCK(cs_main); - const uint256 hash(block.GetHash()); - std::map >::iterator it = mapBlockSource.find(hash); - int nDoS = 0; - if (state.IsInvalid(nDoS)) - { - // Don't send reject message with code 0 or an internal reject code. - if (it != mapBlockSource.end() && State(it->second.first) && state.GetRejectCode() > 0 && - state.GetRejectCode() < REJECT_INTERNAL) - { - CBlockReject reject = { - uint8_t(state.GetRejectCode()), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; - State(it->second.first)->rejects.push_back(reject); - if (nDoS > 0 && it->second.second) - { - Misbehaving(it->second.first, nDoS, state.GetRejectReason()); - } - } - } - if (it != mapBlockSource.end()) - { - mapBlockSource.erase(it); - } -} - bool AlreadyHave(const CInv &inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { switch (inv.type) @@ -906,6 +802,7 @@ bool AlreadyHave(const CInv &inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) pnetMan->getChainActive()->pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } case MSG_BLOCK: + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); return pnetMan->getChainActive()->mapBlockIndex.count(inv.hash); } // Don't know what it is, just say we already got one @@ -934,29 +831,32 @@ void static ProcessGetData(CNode *pfrom, CConnman &connman, const Consensus::Par if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) { bool send = false; - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(inv.hash); - if (pnetMan->getChainActive()->chainActive.Contains(mi->second)) - { - send = true; - } - else + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(inv.hash); + if (pindex) { - static const int nOneMonth = 30 * 24 * 60 * 60; - // To prevent fingerprinting attacks, only send blocks - // outside of the active chain if they are valid, and no - // more than a month older (both in time, and in best - // equivalent proof of work) than the best header chain - // we know about. - send = mi->second->IsValid(BLOCK_VALID_SCRIPTS) && - (pnetMan->getChainActive()->pindexBestHeader != nullptr) && - (pnetMan->getChainActive()->pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() < - nOneMonth) && - (GetBlockProofEquivalentTime(*pnetMan->getChainActive()->pindexBestHeader, *mi->second, - *pnetMan->getChainActive()->pindexBestHeader, consensusParams) < nOneMonth); - if (!send) + if (pnetMan->getChainActive()->chainActive.Contains(pindex)) + { + send = true; + } + else { - LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", - __func__, pfrom->GetId()); + static const int nOneMonth = 30 * 24 * 60 * 60; + // To prevent fingerprinting attacks, only send blocks + // outside of the active chain if they are valid, and no + // more than a month older (both in time, and in best + // equivalent proof of work) than the best header chain + // we know about. + send = pindex->IsValid(BLOCK_VALID_SCRIPTS) && + (pnetMan->getChainActive()->pindexBestHeader != nullptr) && + (pnetMan->getChainActive()->pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() < + nOneMonth) && + (GetBlockProofEquivalentTime(*pnetMan->getChainActive()->pindexBestHeader, *pindex, + *pnetMan->getChainActive()->pindexBestHeader, consensusParams) < nOneMonth); + if (!send) + { + LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", + __func__, pfrom->GetId()); + } } } // Disconnect node in case we have reached the outbound limit @@ -966,7 +866,7 @@ void static ProcessGetData(CNode *pfrom, CConnman &connman, const Consensus::Par static const int nOneWeek = 7 * 24 * 60 * 60; if (send && connman.OutboundTargetReached(true) && (((pnetMan->getChainActive()->pindexBestHeader != nullptr) && - (pnetMan->getChainActive()->pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > + (pnetMan->getChainActive()->pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() > nOneWeek)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) @@ -979,11 +879,11 @@ void static ProcessGetData(CNode *pfrom, CConnman &connman, const Consensus::Par } // Pruned nodes may have deleted the block, so check whether // it's available before trying to send. - if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) + if (send && (pindex->nStatus & BLOCK_HAVE_DATA)) { // Send block from disk CBlock block; - if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) + if (!ReadBlockFromDisk(block, pindex, consensusParams)) { LogPrintf("cannot load block from disk"); assert(false); @@ -1139,7 +1039,6 @@ bool static ProcessMessage(CNode *pfrom, { connman.PushMessage( pfrom, NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, std::string("Duplicate version message")); - LOCK(cs_main); Misbehaving(pfrom, 1, "multiple-version"); return false; } @@ -1242,10 +1141,7 @@ bool static ProcessMessage(CNode *pfrom, pfrom->nVersion = nVersion; // Potentially mark this peer as a preferred download peer. - { - LOCK(cs_main); - UpdatePreferredDownload(pfrom, State(pfrom->GetId())); - } + UpdatePreferredDownload(pfrom); if (!pfrom->fInbound) { @@ -1304,7 +1200,6 @@ bool static ProcessMessage(CNode *pfrom, else if (pfrom->nVersion == 0) { // Must have a version message before anything else - LOCK(cs_main); Misbehaving(pfrom, 1, "missing-version"); return false; } @@ -1318,8 +1213,8 @@ bool static ProcessMessage(CNode *pfrom, { // Mark this node as currently connected, so we update its timestamp // later. - LOCK(cs_main); - State(pfrom->GetId())->fCurrentlyConnected = true; + CNodeStateAccessor state(nodestateman, pfrom->GetId()); + state->fCurrentlyConnected = true; } if (pfrom->nVersion >= SENDHEADERS_VERSION) @@ -1337,7 +1232,6 @@ bool static ProcessMessage(CNode *pfrom, { { // Must have a verack message before anything else - LOCK(cs_main); Misbehaving(pfrom, 1, "missing-verack"); } { @@ -1362,7 +1256,6 @@ bool static ProcessMessage(CNode *pfrom, } if (vAddr.size() > 1000) { - LOCK(cs_main); Misbehaving(pfrom, 20, "oversized-addr"); return error("message addr size() = %u", vAddr.size()); } @@ -1408,8 +1301,8 @@ bool static ProcessMessage(CNode *pfrom, else if (strCommand == NetMsgType::SENDHEADERS) { - LOCK(cs_main); - State(pfrom->GetId())->fPreferHeaders = true; + CNodeStateAccessor state(nodestateman, pfrom->GetId()); + state->fPreferHeaders = true; } @@ -1419,7 +1312,6 @@ bool static ProcessMessage(CNode *pfrom, vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { - LOCK(cs_main); Misbehaving(pfrom, 20, "oversized-inv"); return error("message inv size() = %u", vInv.size()); } @@ -1466,8 +1358,8 @@ bool static ProcessMessage(CNode *pfrom, connman.PushMessage(pfrom, NetMsgType::GETHEADERS, pnetMan->getChainActive()->chainActive.GetLocator(pnetMan->getChainActive()->pindexBestHeader), inv.hash); - CNodeState *nodestate = State(pfrom->GetId()); - if (CanDirectFetch(chainparams.GetConsensus()) && + CNodeStateAccessor nodestate(nodestateman, pfrom->GetId()); + if (CanDirectFetch(chainparams.GetConsensus()) && nodestate.IsNull() == false && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { vToFetch.push_back(inv); @@ -1511,7 +1403,6 @@ bool static ProcessMessage(CNode *pfrom, vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { - LOCK(cs_main); Misbehaving(pfrom, 20, "too-many-inv"); return error("message getdata size() = %u", vInv.size()); } @@ -1534,23 +1425,6 @@ bool static ProcessMessage(CNode *pfrom, uint256 hashStop; vRecv >> locator >> hashStop; - // We might have announced the currently-being-connected tip using a - // compact block, which resulted in the peer sending a getblocks - // request, which we would otherwise respond to without the new block. - // To avoid this situation we simply verify that we are on our best - // known chain now. This is super overkill, but we handle it better - // for getheaders requests, and there are no known nodes which support - // compact blocks but still use getblocks to request blocks. - { - std::shared_ptr a_recent_block; - { - LOCK(cs_most_recent_block); - a_recent_block = most_recent_block; - } - CValidationState dummy; - ActivateBestChain(dummy, pnetMan->getActivePaymentNetwork(), a_recent_block); - } - LOCK(cs_main); // Find the last block the caller has in the main chain @@ -1599,17 +1473,16 @@ bool static ProcessMessage(CNode *pfrom, return true; } - CNodeState *nodestate = State(pfrom->GetId()); + CNodeStateAccessor nodestate(nodestateman, pfrom->GetId()); CBlockIndex *pindex = nullptr; if (locator.IsNull()) { // If locator is null, return the hashStop block - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hashStop); - if (mi == pnetMan->getChainActive()->mapBlockIndex.end()) + pindex = pnetMan->getChainActive()->LookupBlockIndex(hashStop); + if (!pindex) { return true; } - pindex = (*mi).second; } else { @@ -1878,7 +1751,6 @@ bool static ProcessMessage(CNode *pfrom, unsigned int nCount = ReadCompactSize(vRecv); if (nCount > MAX_HEADERS_RESULTS) { - LOCK(cs_main); Misbehaving(pfrom->GetId(), 20, "too-many-headers"); return error("headers message size = %u", nCount); } @@ -1934,7 +1806,7 @@ bool static ProcessMessage(CNode *pfrom, } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); - CNodeState *nodestate = State(pfrom->GetId()); + CNodeStateAccessor nodestate(nodestateman, pfrom->GetId()); // If this set of headers is valid and ends in a block with at least as // much work as our tip, download as much as possible. if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && @@ -1994,17 +1866,17 @@ bool static ProcessMessage(CNode *pfrom, else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing { - std::shared_ptr pblock = std::make_shared(); - vRecv >> *pblock; + CBlock block; + vRecv >> block; - LogPrint("net", "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->id); + const uint256 hash(block.GetHash()); + LogPrint("net", "received block %s peer=%d\n", hash.ToString(), pfrom->id); // Process all blocks from whitelisted peers, even if not requested, // unless we're still syncing with the network. Such an unrequested // block may still be processed, subject to the conditions in // AcceptBlock(). bool forceProcessing = pfrom->fWhitelisted && !pnetMan->getChainActive()->IsInitialBlockDownload(); - const uint256 hash(pblock->GetHash()); { LOCK(cs_main); // Also always process if we requested the block explicitly, as we @@ -2016,7 +1888,7 @@ bool static ProcessMessage(CNode *pfrom, mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); } CValidationState state; - ProcessNewBlock(state, chainparams, pfrom, pblock, forceProcessing, NULL); + ProcessNewBlock(state, chainparams, pfrom, &block, forceProcessing, NULL); int nDoS; if (state.IsInvalid(nDoS)) { @@ -2025,7 +1897,6 @@ bool static ProcessMessage(CNode *pfrom, state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash); if (nDoS > 0) { - LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS, "invalid-blk"); } } @@ -2468,7 +2339,11 @@ bool SendMessages(CNode *pto, CConnman &connman) { return true; } - CNodeState &state = *State(pto->GetId()); + CNodeStateAccessor nodestate(nodestateman, pto->GetId()); + if (nodestate.IsNull()) + { + return true; + } // Address refresh broadcast int64_t nNow = GetTimeMicros(); @@ -2521,16 +2396,16 @@ bool SendMessages(CNode *pto, CConnman &connman) // Download if this is a nice peer, or we have no nice peers and this one // might do. - bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); + bool fFetch = nodestate->fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); - if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) + if (!nodestate->fSyncStarted && !pto->fClient && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close // to today. if ((nSyncStarted == 0 && fFetch) || pnetMan->getChainActive()->pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { - state.fSyncStarted = true; + nodestate->fSyncStarted = true; nSyncStarted++; const CBlockIndex *pindexStart = pnetMan->getChainActive()->pindexBestHeader; /** @@ -2572,7 +2447,7 @@ bool SendMessages(CNode *pto, CConnman &connman) // or if the peer doesn't want headers, just add all to the inv queue. LOCK(pto->cs_inventory); std::vector vHeaders; - bool fRevertToInv = ((!state.fPreferHeaders && (pto->vBlockHashesToAnnounce.size() > 1)) || + bool fRevertToInv = ((!nodestate->fPreferHeaders && (pto->vBlockHashesToAnnounce.size() > 1)) || pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); // last header queued for delivery CBlockIndex *pBestIndex = nullptr; @@ -2587,9 +2462,8 @@ bool SendMessages(CNode *pto, CConnman &connman) // aren't on chainActive, give up. for (const uint256 &hash : pto->vBlockHashesToAnnounce) { - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hash); - assert(mi != pnetMan->getChainActive()->mapBlockIndex.end()); - CBlockIndex *pindex = mi->second; + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + assert(pindex); if (pnetMan->getChainActive()->chainActive[pindex->nHeight] != pindex) { // Bail out if we reorged away from this block @@ -2616,12 +2490,12 @@ bool SendMessages(CNode *pto, CConnman &connman) // add this to the headers message vHeaders.push_back(pindex->GetBlockHeader()); } - else if (PeerHasHeader(&state, pindex)) + else if (PeerHasHeader(nodestate.Get(), pindex)) { // Keep looking for the first new block. continue; } - else if (pindex->pprev == nullptr || PeerHasHeader(&state, pindex->pprev)) + else if (pindex->pprev == nullptr || PeerHasHeader(nodestate.Get(), pindex->pprev)) { // Peer doesn't have this header but they do have the prior // one. @@ -2640,7 +2514,7 @@ bool SendMessages(CNode *pto, CConnman &connman) } if (!fRevertToInv && !vHeaders.empty()) { - if (state.fPreferHeaders) + if (nodestate->fPreferHeaders) { if (vHeaders.size() > 1) { @@ -2653,7 +2527,7 @@ bool SendMessages(CNode *pto, CConnman &connman) vHeaders.front().GetHash().ToString(), pto->id); } connman.PushMessage(pto, NetMsgType::HEADERS, vHeaders); - state.pindexBestHeaderSent = pBestIndex; + nodestate->pindexBestHeaderSent = pBestIndex; } else { @@ -2668,9 +2542,8 @@ bool SendMessages(CNode *pto, CConnman &connman) if (!pto->vBlockHashesToAnnounce.empty()) { const uint256 &hashToAnnounce = pto->vBlockHashesToAnnounce.back(); - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hashToAnnounce); - assert(mi != pnetMan->getChainActive()->mapBlockIndex.end()); - CBlockIndex *pindex = mi->second; + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hashToAnnounce); + assert(pindex); // Warn if we're announcing a block that is not on the main // chain. This should be very rare and could be optimized out. @@ -2682,7 +2555,7 @@ bool SendMessages(CNode *pto, CConnman &connman) } // If the peer's chain has this block, don't inv it back. - if (!PeerHasHeader(&state, pindex)) + if (!PeerHasHeader(nodestate.Get(), pindex)) { pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); LogPrint("net", "%s: sending inv peer=%d hash=%s\n", __func__, pto->id, hashToAnnounce.ToString()); @@ -2784,16 +2657,6 @@ bool SendMessages(CNode *pto, CConnman &connman) // Detect whether we're stalling nNow = GetTimeMicros(); - if (state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) - { - // Stalling only triggers when the block download window cannot move. - // During normal steady state, the download window should be much larger - // than the to-be-downloaded set of blocks, so disconnection should only - // happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); - pto->fDisconnect = true; - return true; - } // In case there is a block that has been in flight from this peer for 2 + // 0.5 * N times the block interval (with N the number of peers from which // we're downloading validated blocks), disconnect due to timeout. We @@ -2801,16 +2664,17 @@ bool SendMessages(CNode *pto, CConnman &connman) // downstream link being saturated. We only count validated in-flight blocks // so peers can't advertise non-existing block hashes to unreasonably // increase our timeout. - if (state.vBlocksInFlight.size() > 0) + if (nodestate->vBlocksInFlight.size() > 0) { int64_t targetSpacing = consensusParams.nTargetSpacing; if (pnetMan->getChainActive()->chainActive.Tip()->GetMedianTimePast() > SERVICE_UPGRADE_HARDFORK) { targetSpacing = 150; } - QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); - int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); - if (nNow > state.nDownloadingSince + + const QueuedBlock &queuedBlock = nodestate->vBlocksInFlight.front(); + int nOtherPeersWithValidatedDownloads = + nPeersWithValidatedDownloads - (nodestate->nBlocksInFlightValidHeaders > 0); + if (nNow > nodestate->nDownloadingSince + targetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { @@ -2827,12 +2691,12 @@ bool SendMessages(CNode *pto, CConnman &connman) // std::vector vGetData; if (!pto->fClient && (fFetch || !pnetMan->getChainActive()->IsInitialBlockDownload()) && - state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) + nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector vToDownload; NodeId staller = -1; FindNextBlocksToDownload( - pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); + pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - nodestate->nBlocksInFlight, vToDownload, staller); for (const CBlockIndex *pindex : vToDownload) { uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); @@ -2841,14 +2705,6 @@ bool SendMessages(CNode *pto, CConnman &connman) LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), pindex->nHeight, pto->id); } - if (state.nBlocksInFlight == 0 && staller != -1) - { - if (State(staller)->nStallingSince == 0) - { - State(staller)->nStallingSince = nNow; - LogPrint("net", "Stall started peer=%d\n", staller); - } - } } // diff --git a/src/net/messages.h b/src/net/messages.h index 219f8f4c..0bfd5aed 100644 --- a/src/net/messages.h +++ b/src/net/messages.h @@ -63,12 +63,8 @@ class PeerLogicValidation : public CValidationInterface public: PeerLogicValidation(CConnman *connmanIn); - void BlockConnected(const std::shared_ptr &pblock, - const CBlockIndex *pindexConnected, - const std::vector &vtxConflicted) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; - void BlockChecked(const CBlock &block, const CValidationState &state) override; - void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr &pblock) override; + void NewPoWValidBlock(const CBlockIndex *pindex, const CBlock *pblock) override; }; struct CNodeStateStats @@ -98,94 +94,6 @@ struct QueuedBlock extern std::map::iterator> > mapBlocksInFlight; extern std::map > mapBlockSource; - -/** - * Maintain validation-specific state about nodes, protected by cs_main, instead - * by CNode's own locks. This simplifies asynchronous operation, where - * processing of incoming data is done after the ProcessMessage call returns, - * and we're no longer holding the node's locks. - */ -struct CNodeState -{ - //! The peer's address - const CService address; - //! Whether we have a fully established connection. - bool fCurrentlyConnected; - //! Accumulated misbehaviour score for this peer. - int nMisbehavior; - //! Whether this peer should be disconnected and banned (unless - //! whitelisted). - bool fShouldBan; - //! String name of this peer (debugging/logging purposes). - const std::string name; - //! List of asynchronously-determined block rejections to notify this peer - //! about. - std::vector rejects; - //! The best known block we know this peer has announced. - const CBlockIndex *pindexBestKnownBlock; - //! The hash of the last unknown block this peer has announced. - uint256 hashLastUnknownBlock; - //! The last full block we both have. - const CBlockIndex *pindexLastCommonBlock; - //! The best header we have sent our peer. - const CBlockIndex *pindexBestHeaderSent; - //! Length of current-streak of unconnecting headers announcements - int nUnconnectingHeaders; - //! Whether we've started headers synchronization with this peer. - bool fSyncStarted; - //! Since when we're stalling block download progress (in microseconds), or - //! 0. - int64_t nStallingSince; - std::list vBlocksInFlight; - //! When the first entry in vBlocksInFlight started downloading. Don't care - //! when vBlocksInFlight is empty. - int64_t nDownloadingSince; - int nBlocksInFlight; - int nBlocksInFlightValidHeaders; - //! Whether we consider this a preferred download peer. - bool fPreferredDownload; - //! Whether this peer wants invs or headers (when possible) for block - //! announcements. - bool fPreferHeaders; - /** - * Whether this peer will send us cmpctblocks if we request them. - * This is not used to gate request logic, as we really only care about - * fSupportsDesiredCmpctVersion, but is used as a flag to "lock in" the - * version of compact blocks we send. - */ - bool fProvidesHeaderAndIDs; - /** - * If we've announced NODE_WITNESS to this peer: whether the peer sends - * witnesses in cmpctblocks/blocktxns, otherwise: whether this peer sends - * non-witnesses in cmpctblocks/blocktxns. - */ - bool fSupportsDesiredCmpctVersion; - - CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) - { - fCurrentlyConnected = false; - nMisbehavior = 0; - fShouldBan = false; - pindexBestKnownBlock = nullptr; - hashLastUnknownBlock.SetNull(); - pindexLastCommonBlock = nullptr; - pindexBestHeaderSent = nullptr; - nUnconnectingHeaders = 0; - fSyncStarted = false; - nStallingSince = 0; - nDownloadingSince = 0; - nBlocksInFlight = 0; - nBlocksInFlightValidHeaders = 0; - fPreferredDownload = false; - fPreferHeaders = false; - fProvidesHeaderAndIDs = false; - fSupportsDesiredCmpctVersion = false; - } -}; - -extern std::map mapNodeState; -CNodeState *State(NodeId pnode); - /** Get statistics from node state */ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); /** Increase a node's misbehavior score. */ diff --git a/src/net/nodestate.cpp b/src/net/nodestate.cpp new file mode 100644 index 00000000..8b17fc8c --- /dev/null +++ b/src/net/nodestate.cpp @@ -0,0 +1,24 @@ +#include "nodestate.h" + +CNodesStateManager nodestateman; + +CNodeState *CNodesStateManager::_GetNodeState(const NodeId id) +{ + std::map::iterator it = mapNodeState.find(id); + if (it == mapNodeState.end()) + return nullptr; + return &it->second; +} + +void CNodesStateManager::InitializeNodeState(const CNode *pnode) +{ + LOCK(cs); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(pnode->GetId()), + std::forward_as_tuple(pnode->addr, pnode->GetAddrName())); +} + +void CNodesStateManager::RemoveNodeState(const NodeId id) +{ + LOCK(cs); + mapNodeState.erase(id); +} diff --git a/src/net/nodestate.h b/src/net/nodestate.h new file mode 100644 index 00000000..95f46b9e --- /dev/null +++ b/src/net/nodestate.h @@ -0,0 +1,143 @@ +#include "messages.h" +#include "net.h" + +/** + * Maintain validation-specific state about nodes, protected by cs_main, instead + * by CNode's own locks. This simplifies asynchronous operation, where + * processing of incoming data is done after the ProcessMessage call returns, + * and we're no longer holding the node's locks. + */ +struct CNodeState +{ + //! The peer's address + const CService address; + //! Whether we have a fully established connection. + bool fCurrentlyConnected; + //! Accumulated misbehaviour score for this peer. + int nMisbehavior; + //! Whether this peer should be disconnected and banned (unless + //! whitelisted). + bool fShouldBan; + //! String name of this peer (debugging/logging purposes). + const std::string name; + //! List of asynchronously-determined block rejections to notify this peer + //! about. + std::vector rejects; + //! The best known block we know this peer has announced. + const CBlockIndex *pindexBestKnownBlock; + //! The hash of the last unknown block this peer has announced. + uint256 hashLastUnknownBlock; + //! The last full block we both have. + const CBlockIndex *pindexLastCommonBlock; + //! The best header we have sent our peer. + const CBlockIndex *pindexBestHeaderSent; + //! Length of current-streak of unconnecting headers announcements + int nUnconnectingHeaders; + //! Whether we've started headers synchronization with this peer. + bool fSyncStarted; + + std::list vBlocksInFlight; + //! When the first entry in vBlocksInFlight started downloading. Don't care + //! when vBlocksInFlight is empty. + int64_t nDownloadingSince; + int nBlocksInFlight; + int nBlocksInFlightValidHeaders; + //! Whether we consider this a preferred download peer. + bool fPreferredDownload; + //! Whether this peer wants invs or headers (when possible) for block + //! announcements. + bool fPreferHeaders; + /** + * Whether this peer will send us cmpctblocks if we request them. + * This is not used to gate request logic, as we really only care about + * fSupportsDesiredCmpctVersion, but is used as a flag to "lock in" the + * version of compact blocks we send. + */ + bool fProvidesHeaderAndIDs; + /** + * If we've announced NODE_WITNESS to this peer: whether the peer sends + * witnesses in cmpctblocks/blocktxns, otherwise: whether this peer sends + * non-witnesses in cmpctblocks/blocktxns. + */ + bool fSupportsDesiredCmpctVersion; + + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) + { + fCurrentlyConnected = false; + nMisbehavior = 0; + fShouldBan = false; + pindexBestKnownBlock = nullptr; + hashLastUnknownBlock.SetNull(); + pindexLastCommonBlock = nullptr; + pindexBestHeaderSent = nullptr; + nUnconnectingHeaders = 0; + fSyncStarted = false; + nDownloadingSince = 0; + nBlocksInFlight = 0; + nBlocksInFlightValidHeaders = 0; + fPreferredDownload = false; + fPreferHeaders = false; + fProvidesHeaderAndIDs = false; + fSupportsDesiredCmpctVersion = false; + } +}; + +class CNodesStateManager +{ +protected: + CCriticalSection cs; + std::map mapNodeState; + friend class CNodeStateAccessor; + +public: + CNodeState *_GetNodeState(const NodeId id); + + /** Add a nodestate from the map */ + void InitializeNodeState(const CNode *pnode); + + /** Delete a nodestate from the map */ + void RemoveNodeState(const NodeId id); + + /** Clear the entire nodestate map */ + void Clear() + { + LOCK(cs); + mapNodeState.clear(); + } + + /** Is mapNodestate empty */ + bool Empty() + { + LOCK(cs); + return mapNodeState.empty(); + } +}; + +class CNodeStateAccessor +{ + CCriticalSection *cs; + CNodeState *obj; + +public: + CNodeStateAccessor(CCriticalSection *_cs, CNodeState *_obj) : cs(_cs), obj(_obj) { cs->lock(); } + CNodeStateAccessor(CNodesStateManager &ns, const NodeId id) + { + cs = &ns.cs; + cs->lock(); + obj = ns._GetNodeState(id); + } + + CNodeState *operator->() { return obj; } + CNodeState &operator*() { return *obj; } + bool operator!=(void *ptr) { return obj != ptr; } + bool operator==(void *ptr) { return obj == ptr; } + bool IsNull() { return obj == nullptr; } + CNodeState *Get() { return obj; } + ~CNodeStateAccessor() + { + obj = nullptr; + cs->unlock(); + } +}; + +extern CNodesStateManager nodestateman; diff --git a/src/processblock.cpp b/src/processblock.cpp index 28f526a4..85f2f6a5 100644 --- a/src/processblock.cpp +++ b/src/processblock.cpp @@ -75,83 +75,8 @@ class WarningBitsConditionChecker : public AbstractThresholdConditionChecker } }; -struct PerBlockConnectTrace -{ - CBlockIndex *pindex = nullptr; - std::shared_ptr pblock; - std::shared_ptr > conflictedTxs; - PerBlockConnectTrace() : conflictedTxs(std::make_shared >()) {} -}; - -/** - * Used to track blocks whose transactions were applied to the UTXO state as a - * part of a single ActivateBestChainStep call. - * - * This class also tracks transactions that are removed from the mempool as - * conflicts (per block) and can be used to pass all those transactions through - * SyncTransaction. - * - * This class assumes (and asserts) that the conflicted transactions for a given - * block are added via mempool callbacks prior to the BlockConnected() - * associated with those transactions. If any transactions are marked - * conflicted, it is assumed that an associated block will always be added. - * - * This class is single-use, once you call GetBlocksConnected() you have to - * throw it away and make a new one. - */ -class ConnectTrace -{ -private: - std::vector blocksConnected; - CTxMemPool &pool; - -public: - ConnectTrace(CTxMemPool &_pool) : blocksConnected(1), pool(_pool) - { - pool.NotifyEntryRemoved.connect(boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); - } - - ~ConnectTrace() - { - pool.NotifyEntryRemoved.disconnect(boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); - } - - void BlockConnected(CBlockIndex *pindex, std::shared_ptr pblock) - { - assert(!blocksConnected.back().pindex); - assert(pindex); - assert(pblock); - blocksConnected.back().pindex = pindex; - blocksConnected.back().pblock = std::move(pblock); - blocksConnected.emplace_back(); - } - - std::vector &GetBlocksConnected() - { - // We always keep one extra block at the end of our list because blocks - // are added after all the conflicted transactions have been filled in. - // Thus, the last entry should always be an empty one waiting for the - // transactions from the next block. We pop the last entry here to make - // sure the list we return is sane. - assert(!blocksConnected.back().pindex); - assert(blocksConnected.back().conflictedTxs->empty()); - blocksConnected.pop_back(); - return blocksConnected; - } - - void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) - { - assert(!blocksConnected.back().pindex); - if (reason == MemPoolRemovalReason::CONFLICT) - { - blocksConnected.back().conflictedTxs->emplace_back(std::move(txRemoved)); - } - } -}; - - /** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */ -bool AcceptBlock(const std::shared_ptr pblock, +bool AcceptBlock(const CBlock *pblock, CValidationState &state, const CNetworkTemplate &chainparams, CBlockIndex **ppindex, @@ -241,7 +166,7 @@ bool AcceptBlock(const std::shared_ptr pblock, bool ProcessNewBlock(CValidationState &state, const CNetworkTemplate &chainparams, const CNode *pfrom, - const std::shared_ptr pblock, + const CBlock *pblock, bool fForceProcessing, CDiskBlockPos *dbp) { @@ -264,7 +189,6 @@ bool ProcessNewBlock(CValidationState &state, CheckBlockIndex(chainparams.GetConsensus()); if (!ret) { - GetMainSignals().BlockChecked(*pblock, state); return error("%s: AcceptBlock FAILED", __func__); } } @@ -395,7 +319,10 @@ bool DisconnectTip(CValidationState &state, const Consensus::Params &consensusPa UpdateTip(pindexDelete->pprev); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - GetMainSignals().BlockDisconnected(pblock); + for (const auto &ptx : block.vtx) + { + SyncWithWallets(ptx, nullptr, -1); + } return true; } @@ -414,30 +341,22 @@ static int64_t nTimeTotal = 0; bool ConnectTip(CValidationState &state, const CNetworkTemplate &chainparams, CBlockIndex *pindexNew, - const std::shared_ptr &pblock, - ConnectTrace &connectTrace) + const CBlock *pblock) { assert(pindexNew->pprev == pnetMan->getChainActive()->chainActive.Tip()); // Read block from disk. int64_t nTime1 = GetTimeMicros(); - std::shared_ptr pthisBlock; + CBlock block; if (!pblock) { - std::shared_ptr pblockNew = std::make_shared(); - if (!ReadBlockFromDisk(*pblockNew, pindexNew, chainparams.GetConsensus())) + if (!ReadBlockFromDisk(block, pindexNew, chainparams.GetConsensus())) { return AbortNode(state, "Failed to read block"); } - pthisBlock = pblockNew; - } - else - { - pthisBlock = pblock; + pblock = █ } - const CBlock &blockConnecting = *pthisBlock; - // Apply the block atomically to the chain state. int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; @@ -446,8 +365,7 @@ bool ConnectTip(CValidationState &state, "bench", " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { CCoinsViewCache view(pnetMan->getChainActive()->pcoinsTip.get()); - bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); - GetMainSignals().BlockChecked(blockConnecting, state); + bool rv = ConnectBlock(*pblock, state, pindexNew, view); if (!rv) { if (state.IsInvalid()) @@ -475,18 +393,30 @@ bool ConnectTip(CValidationState &state, // Remove conflicting transactions from the mempool. std::list txConflicted; mempool.removeForBlock( - blockConnecting.vtx, pindexNew->nHeight, txConflicted, !pnetMan->getChainActive()->IsInitialBlockDownload()); + pblock->vtx, pindexNew->nHeight, txConflicted, !pnetMan->getChainActive()->IsInitialBlockDownload()); // Update chainActive & related variables. UpdateTip(pindexNew); + // Tell wallet about transactions that went from mempool + // to conflicted: + for (const auto &ptx : txConflicted) + { + SyncWithWallets(ptx, nullptr, -1); + } + // ... and about transactions that got confirmed: + int txIdx = 0; + for (const auto &ptx : pblock->vtx) + { + SyncWithWallets(ptx, pblock, txIdx); + txIdx++; + } + int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint( "bench", " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); LogPrint("bench", "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); - - connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true; } @@ -578,8 +508,7 @@ void CheckForkWarningConditionsOnNewFork(CBlockIndex *pindexNewForkTip) bool ActivateBestChainStep(CValidationState &state, const CNetworkTemplate &chainparams, CBlockIndex *pindexMostWork, - const std::shared_ptr &pblock, - ConnectTrace &connectTrace) + const CBlock *pblock) { AssertLockHeld(cs_main); bool fInvalidFound = false; @@ -617,8 +546,7 @@ bool ActivateBestChainStep(CValidationState &state, // Connect new blocks. BOOST_REVERSE_FOREACH (CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip( - state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : NULL, connectTrace)) + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : nullptr)) { if (state.IsInvalid()) { @@ -671,9 +599,7 @@ bool ActivateBestChainStep(CValidationState &state, * or an activated best chain. pblock is either NULL or a pointer to a block * that is already loaded (to avoid loading it again from disk). */ -bool ActivateBestChain(CValidationState &state, - const CNetworkTemplate &chainparams, - const std::shared_ptr pblock) +bool ActivateBestChain(CValidationState &state, const CNetworkTemplate &chainparams, const CBlock *pblock) { CBlockIndex *pindexMostWork = NULL; do @@ -692,29 +618,22 @@ bool ActivateBestChain(CValidationState &state, { LOCK(cs_main); - // Destructed before cs_main is unlocked. - ConnectTrace connectTrace(mempool); - CBlockIndex *pindexOldTip = pnetMan->getChainActive()->chainActive.Tip(); pindexMostWork = FindMostWorkChain(); // Whether we have anything to do at all. - if (pindexMostWork == NULL || pindexMostWork == pnetMan->getChainActive()->chainActive.Tip()) + if (pindexMostWork == nullptr || pindexMostWork == pnetMan->getChainActive()->chainActive.Tip()) return true; if (!ActivateBestChainStep(state, chainparams, pindexMostWork, - pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL, connectTrace)) + pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullptr)) + { return false; + } pindexNewTip = pnetMan->getChainActive()->chainActive.Tip(); pindexFork = pnetMan->getChainActive()->chainActive.FindFork(pindexOldTip); fInitialDownload = pnetMan->getChainActive()->IsInitialBlockDownload(); - - for (const PerBlockConnectTrace &trace : connectTrace.GetBlocksConnected()) - { - assert(trace.pblock && trace.pindex); - GetMainSignals().BlockConnected(trace.pblock, trace.pindex, *trace.conflictedTxs); - } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -779,6 +698,7 @@ void CheckBlockIndex(const Consensus::Params &consensusParams) } LOCK(cs_main); + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain, // so we have the genesis block in mapBlockIndex but no active chain. (A few of the tests when @@ -1669,93 +1589,3 @@ bool DisconnectBlock(const CBlock &block, return fClean; } - -/** Comparison function for sorting the getchaintips heads. */ -struct CompareBlocksByHeight -{ - bool operator()(const CBlockIndex *a, const CBlockIndex *b) const - { - /* Make sure that unequal blocks with the same height do not compare - equal. Use the pointers themselves to make a distinction. */ - - if (a->nHeight != b->nHeight) - return (a->nHeight > b->nHeight); - - return a < b; - } -}; - -void removeImpossibleChainTips() -{ - LOCK(cs_main); - int deletionCount = 0; - std::set setTips; - for (auto &item : pnetMan->getChainActive()->mapBlockIndex) - setTips.insert(item.second); - for (auto &item : pnetMan->getChainActive()->mapBlockIndex) - { - CBlockIndex *pprev = item.second->pprev; - if (pprev) - setTips.erase(pprev); - } - - setTips.insert(pnetMan->getChainActive()->chainActive.Tip()); - - const int currentHeight = pnetMan->getChainActive()->chainActive.Height(); - for (CBlockIndex *block : setTips) - { - const int forkHeight = pnetMan->getChainActive()->chainActive.FindFork(block)->nHeight; - const int branchLen = block->nHeight - forkHeight; - std::string status = ""; - if (pnetMan->getChainActive()->chainActive.Contains(block)) - { - // This block is part of the currently active chain. - status = "active"; - } - else if (block->nStatus & BLOCK_FAILED_MASK) - { - // This block or one of its ancestors is invalid. - status = "invalid"; - } - else if (block->nChainTx == 0) - { - // This block cannot be connected because full block data for it or one of its parents is missing. - status = "headers-only"; - } - else if (block->IsValid(BLOCK_VALID_SCRIPTS)) - { - // This block is fully validated, but no longer part of the active chain. It was probably the active block - // once, but was reorganized. - status = "valid-fork"; - } - else if (block->IsValid(BLOCK_VALID_TREE)) - { - // The headers for this block are valid, but it has not been validated. It was probably never part of the - // most-work chain. - status = "valid-headers"; - } - else - { - // No clue. - status = "unknown"; - } - // after 30 blocks we cannot re-org anyway so after 100 it is definitely safe to delete data - if (status != "active" && status != "unknown" && forkHeight <= currentHeight - 100 && branchLen <= 50) - { - CBlockIndex *curBlock = block; - while (curBlock->nHeight > forkHeight) - { - pnetMan->getChainActive()->pblocktree->EraseBlockIndex(curBlock->GetBlockHash()); - pnetMan->getChainActive()->mapBlockIndex.erase(curBlock->GetBlockHash()); - LogPrintf("cleaning up index %s \n", curBlock->GetBlockHash().ToString().c_str()); - deletionCount++; - curBlock = curBlock->pprev; - } - } - } - /// only print that we deleted if we did delete something - if (deletionCount > 0) - { - LogPrintf("found %i impossible indexes and deleted them \n", deletionCount); - } -} diff --git a/src/processblock.h b/src/processblock.h index e904b852..8886e3b0 100644 --- a/src/processblock.h +++ b/src/processblock.h @@ -37,10 +37,28 @@ extern bool fLargeWorkInvalidChainFound; CBlockIndex *FindMostWorkChain(); void CheckBlockIndex(const Consensus::Params &consensusParams); +/** + * Process an incoming block. This only returns after the best known valid + * block is made active. Note that it does not, however, guarantee that the + * specific block passed to it has been checked for validity! + * + * @param[out] state This may be set to an Error state if any error occurred processing it, including during + * validation/connection/etc of otherwise unrelated blocks during reorganisation; or it may be set to an Invalid state + * if pblock is itself invalid (but this is not guaranteed even when the block is checked). If you want to *possibly* + * get feedback on whether pblock is valid, you must also install a CValidationInterface (see validationinterface.h) - + * this will have its BlockChecked method called whenever *any* block completes validation. + * @param[in] pfrom The node which we are receiving the block from; it is added to mapBlockSource and may be + * penalised if the block is invalid. + * @param[in] pblock The block we want to process. + * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and + * whitelisted peers. + * @param[out] dbp If pblock is stored to disk (or already there), this will be set to its location. + * @return True if state.IsValid() + */ bool ProcessNewBlock(CValidationState &state, const CNetworkTemplate &chainparams, const CNode *pfrom, - const std::shared_ptr pblock, + const CBlock *pblock, bool fForceProcessing, CDiskBlockPos *dbp); bool DisconnectTip(CValidationState &state, const Consensus::Params &consensusParams); @@ -67,14 +85,10 @@ bool DisconnectBlock(const CBlock &block, CValidationState &state, const CBlockIndex *pindex, CCoinsViewCache &coins, - bool *pfClean = NULL); - -void removeImpossibleChainTips(); + bool *pfClean = nullptr); /** Find the best known block, and make it the tip of the block chain */ -bool ActivateBestChain(CValidationState &state, - const CNetworkTemplate &chainparams, - const std::shared_ptr pblock = std::shared_ptr()); +bool ActivateBestChain(CValidationState &state, const CNetworkTemplate &chainparams, const CBlock *pblock = nullptr); #endif // PROCESSBLOCK_H diff --git a/src/processheader.cpp b/src/processheader.cpp index 7bb5b907..f39b596c 100644 --- a/src/processheader.cpp +++ b/src/processheader.cpp @@ -33,14 +33,12 @@ bool AcceptBlockHeader(const CBlockHeader &block, AssertLockHeld(cs_main); // Check for duplicate uint256 hash = block.GetHash(); - BlockMap::iterator miSelf = pnetMan->getChainActive()->mapBlockIndex.find(hash); - CBlockIndex *pindex = NULL; + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); if (hash != chainparams.GetConsensus().hashGenesisBlock) { - if (miSelf != pnetMan->getChainActive()->mapBlockIndex.end()) + if (pindex) { // Block header is already known. - pindex = miSelf->second; if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) @@ -52,11 +50,11 @@ bool AcceptBlockHeader(const CBlockHeader &block, return false; // Get prev block index - CBlockIndex *pindexPrev = NULL; - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(block.hashPrevBlock); - if (mi == pnetMan->getChainActive()->mapBlockIndex.end()) + CBlockIndex *pindexPrev = pnetMan->getChainActive()->LookupBlockIndex(block.hashPrevBlock); + if (!pindexPrev) + { return state.DoS(10, error("%s: prev block not found", __func__), 0, "bad-prevblk"); - pindexPrev = (*mi).second; + } if (pindexPrev->nStatus & BLOCK_FAILED_MASK) return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); @@ -67,11 +65,15 @@ bool AcceptBlockHeader(const CBlockHeader &block, if (!ContextualCheckBlockHeader(block, state, pindexPrev)) return false; } - if (pindex == NULL) + if (pindex == nullptr) + { pindex = pnetMan->getChainActive()->AddToBlockIndex(block); + } if (ppindex) + { *ppindex = pindex; + } return true; } diff --git a/src/rest.cpp b/src/rest.cpp index 5de7d781..ec0ae8f1 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -154,8 +154,7 @@ static bool rest_headers(HTTPRequest *req, const std::string &strURIPart) headers.reserve(count); { LOCK(cs_main); - auto it = pnetMan->getChainActive()->mapBlockIndex.find(hash); - const CBlockIndex *pindex = (it != pnetMan->getChainActive()->mapBlockIndex.end()) ? it->second : NULL; + const CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); while (pindex != NULL && pnetMan->getChainActive()->chainActive.Contains(pindex)) { headers.push_back(pindex); @@ -225,11 +224,10 @@ static bool rest_block(HTTPRequest *req, const std::string &strURIPart, bool sho CBlockIndex *pblockindex = NULL; { LOCK(cs_main); - if (pnetMan->getChainActive()->mapBlockIndex.count(hash) == 0) + CBlockIndex *pblockindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (!pblockindex) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); - pblockindex = pnetMan->getChainActive()->mapBlockIndex[hash]; - if (!ReadBlockFromDisk(block, pblockindex, pnetMan->getActivePaymentNetwork()->GetConsensus())) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -541,7 +539,8 @@ static bool rest_getutxos(HTTPRequest *req, const std::string &strURIPart) std::vector hits; bitmap.resize((vOutPoints.size() + 7) / 8); { - LOCK2(cs_main, mempool.cs); + LOCK(cs_main); + READLOCK(mempool.cs); CCoinsView viewDummy; CCoinsViewCache view(&viewDummy); diff --git a/src/rpc/rpcblockchain.cpp b/src/rpc/rpcblockchain.cpp index d1b11fb0..49b5ffe4 100644 --- a/src/rpc/rpcblockchain.cpp +++ b/src/rpc/rpcblockchain.cpp @@ -43,6 +43,8 @@ #include +extern CNetworkManager *pnetMan; + extern void TxToJSON(const CTransaction &tx, const uint256 hashBlock, UniValue &entry); void ScriptPubKeyToJSON(const CScript &scriptPubKey, UniValue &out, bool fIncludeHex); @@ -201,7 +203,7 @@ UniValue mempoolToJSON(bool fVerbose = false) { if (fVerbose) { - LOCK(mempool.cs); + READLOCK(mempool.cs); UniValue o(UniValue::VOBJ); for (auto const &e : mempool.mapTx) { @@ -364,11 +366,10 @@ UniValue getblockheader(const UniValue ¶ms, bool fHelp) if (params.size() > 1) fVerbose = params[1].get_bool(); - if (pnetMan->getChainActive()->mapBlockIndex.count(hash) == 0) + CBlockIndex *pblockindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (!pblockindex) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - CBlockIndex *pblockindex = pnetMan->getChainActive()->mapBlockIndex[hash]; - if (!fVerbose) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); @@ -429,11 +430,11 @@ UniValue getblock(const UniValue ¶ms, bool fHelp) if (params.size() > 1) fVerbose = params[1].get_bool(); - if (pnetMan->getChainActive()->mapBlockIndex.count(hash) == 0) + CBlockIndex *pblockindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (!pblockindex) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); CBlock block; - CBlockIndex *pblockindex = pnetMan->getChainActive()->mapBlockIndex[hash]; if (!ReadBlockFromDisk(block, pblockindex, pnetMan->getActivePaymentNetwork()->GetConsensus())) throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); @@ -479,8 +480,7 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = pcursor->GetBestBlock(); { - LOCK(cs_main); - stats.nHeight = pnetMan->getChainActive()->mapBlockIndex.find(stats.hashBlock)->second->nHeight; + stats.nHeight = pnetMan->getChainActive()->LookupBlockIndex(stats.hashBlock)->nHeight; } ss << stats.hashBlock; uint256 prevkey; @@ -606,7 +606,7 @@ UniValue gettxout(const UniValue ¶ms, bool fHelp) Coin coin; if (fMempool) { - LOCK(mempool.cs); + READLOCK(mempool.cs); CCoinsViewMemPool view(pnetMan->getChainActive()->pcoinsTip.get(), mempool); if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { @@ -621,9 +621,8 @@ UniValue gettxout(const UniValue ¶ms, bool fHelp) } } - BlockMap::iterator it = - pnetMan->getChainActive()->mapBlockIndex.find(pnetMan->getChainActive()->pcoinsTip->GetBestBlock()); - CBlockIndex *pindex = it->second; + CBlockIndex *pindex = + pnetMan->getChainActive()->LookupBlockIndex(pnetMan->getChainActive()->pcoinsTip->GetBestBlock()); ret.push_back(Pair("bestblock", pindex->GetBlockHash().GetHex())); if ((unsigned int)coin.nHeight == MEMPOOL_HEIGHT) ret.push_back(Pair("confirmations", 0)); @@ -818,6 +817,46 @@ struct CompareBlocksByHeight } }; +static std::set GetChainTips() +{ + /* + * Idea: the set of chain tips is chainActive.tip, plus orphan blocks which do not have another orphan building off + * of them. + * Algorithm: + * - Make one pass through mapBlockIndex, picking out the orphan blocks, and also storing a set of the orphan + * block's pprev pointers. + * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. + * - add chainActive.Tip() + */ + std::set setTips; + std::set setOrphans; + std::set setPrevs; + + AssertLockHeld(cs_main); // for chainActive + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); + for (const std::pair &item : pnetMan->getChainActive()->mapBlockIndex) + { + if (!pnetMan->getChainActive()->chainActive.Contains(item.second)) + { + setOrphans.insert(item.second); + setPrevs.insert(item.second->pprev); + } + } + + for (auto &it : setOrphans) + { + if (setPrevs.erase(it) == 0) + { + setTips.insert(it); + } + } + + // Always report the currently active tip. + setTips.insert(pnetMan->getChainActive()->chainActive.Tip()); + + return setTips; +} + UniValue getchaintips(const UniValue ¶ms, bool fHelp) { if (fHelp || params.size() != 0) @@ -853,21 +892,8 @@ UniValue getchaintips(const UniValue ¶ms, bool fHelp) LOCK(cs_main); - /* Build up a list of chain tips. We start with the list of all - known blocks, and successively remove blocks that appear as pprev - of another block. */ - std::set setTips; - for (auto const &item : pnetMan->getChainActive()->mapBlockIndex) - setTips.insert(item.second); - for (auto const &item : pnetMan->getChainActive()->mapBlockIndex) - { - const CBlockIndex *pprev = item.second->pprev; - if (pprev) - setTips.erase(pprev); - } - - // Always report the currently active tip. - setTips.insert(pnetMan->getChainActive()->chainActive.Tip()); + std::set setTips; + setTips = GetChainTips(); /* Construct the output array. */ UniValue res(UniValue::VARR); @@ -970,11 +996,11 @@ UniValue invalidateblock(const UniValue ¶ms, bool fHelp) CValidationState state; { - LOCK(cs_main); - if (pnetMan->getChainActive()->mapBlockIndex.count(hash) == 0) + CBlockIndex *pblockindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (!pblockindex) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - CBlockIndex *pblockindex = pnetMan->getChainActive()->mapBlockIndex[hash]; + LOCK(cs_main); InvalidateBlock(state, pnetMan->getActivePaymentNetwork()->GetConsensus(), pblockindex); } @@ -1008,12 +1034,12 @@ UniValue reconsiderblock(const UniValue ¶ms, bool fHelp) uint256 hash(uint256S(strHash)); CValidationState state; + CBlockIndex *pblockindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (!pblockindex) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + { LOCK(cs_main); - if (pnetMan->getChainActive()->mapBlockIndex.count(hash) == 0) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - - CBlockIndex *pblockindex = pnetMan->getChainActive()->mapBlockIndex[hash]; ReconsiderBlock(state, pblockindex); } diff --git a/src/rpc/rpcmining.cpp b/src/rpc/rpcmining.cpp index 80a2341d..e439b660 100644 --- a/src/rpc/rpcmining.cpp +++ b/src/rpc/rpcmining.cpp @@ -208,7 +208,6 @@ UniValue generateBlocks(boost::shared_ptr coinbaseScript, } CValidationState state; - const std::shared_ptr spblock = std::make_shared(*pblock); for (auto tx : pblock->vtx) { LogPrintf("transaction: %s \n", tx->GetHash().GetHex().c_str()); @@ -217,7 +216,7 @@ UniValue generateBlocks(boost::shared_ptr coinbaseScript, LogPrintf("generated %s\n", FormatMoney(vout.nValue).c_str()); } } - if (!ProcessNewBlock(state, pnetMan->getActivePaymentNetwork(), nullptr, spblock, true, nullptr)) + if (!ProcessNewBlock(state, pnetMan->getActivePaymentNetwork(), nullptr, pblock, true, nullptr)) throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); ++nHeight; blockHashes.push_back(pblock->GetHash().GetHex()); @@ -655,10 +654,9 @@ UniValue getblocktemplate(const UniValue ¶ms, bool fHelp) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); uint256 hash = block.GetHash(); - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hash); - if (mi != pnetMan->getChainActive()->mapBlockIndex.end()) + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (pindex) { - CBlockIndex *pindex = mi->second; if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) return "duplicate"; if (pindex->nStatus & BLOCK_FAILED_MASK) @@ -983,10 +981,9 @@ UniValue getposblocktemplate(const UniValue ¶ms, bool fHelp) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); uint256 hash = block.GetHash(); - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hash); - if (mi != pnetMan->getChainActive()->mapBlockIndex.end()) + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (pindex) { - CBlockIndex *pindex = mi->second; if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) return "duplicate"; if (pindex->nStatus & BLOCK_FAILED_MASK) @@ -1253,11 +1250,9 @@ UniValue submitblock(const UniValue ¶ms, bool fHelp) uint256 hash = block.GetHash(); bool fBlockPresent = false; { - LOCK(cs_main); - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hash); - if (mi != pnetMan->getChainActive()->mapBlockIndex.end()) + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (pindex) { - CBlockIndex *pindex = mi->second; if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) return "duplicate"; if (pindex->nStatus & BLOCK_FAILED_MASK) @@ -1270,7 +1265,7 @@ UniValue submitblock(const UniValue ¶ms, bool fHelp) CValidationState state; submitblock_StateCatcher sc(block.GetHash()); RegisterValidationInterface(&sc); - const std::shared_ptr spblock(&block); + const CBlock *spblock(&block); bool fAccepted = ProcessNewBlock(state, pnetMan->getActivePaymentNetwork(), NULL, spblock, true, NULL); UnregisterValidationInterface(&sc); if (fBlockPresent) diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 5c911919..7e453403 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -113,10 +113,9 @@ void TxToJSON(const CTransaction &tx, const uint256 hashBlock, UniValue &entry) if (!hashBlock.IsNull()) { entry.push_back(Pair("blockhash", hashBlock.GetHex())); - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hashBlock); - if (mi != pnetMan->getChainActive()->mapBlockIndex.end() && (*mi).second) + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hashBlock); + if (pindex) { - CBlockIndex *pindex = (*mi).second; if (pnetMan->getChainActive()->chainActive.Contains(pindex)) { entry.push_back( @@ -265,6 +264,7 @@ UniValue gettxoutproof(const UniValue ¶ms, bool fHelp) uint256 hashBlock; if (params.size() > 1) { + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); hashBlock = uint256S(params[1].get_str()); if (!pnetMan->getChainActive()->mapBlockIndex.count(hashBlock)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); @@ -285,6 +285,7 @@ UniValue gettxoutproof(const UniValue ¶ms, bool fHelp) if (!GetTransaction(oneTxid, tx, pnetMan->getActivePaymentNetwork()->GetConsensus(), hashBlock, false) || hashBlock.IsNull()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); if (!pnetMan->getChainActive()->mapBlockIndex.count(hashBlock)) throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); pblockindex = pnetMan->getChainActive()->mapBlockIndex[hashBlock]; @@ -337,12 +338,13 @@ UniValue verifytxoutproof(const UniValue ¶ms, bool fHelp) if (merkleBlock.txn.ExtractMatches(vMatch) != merkleBlock.header.hashMerkleRoot) return res; - LOCK(cs_main); + auto *pindex = pnetMan->getChainActive()->LookupBlockIndex(merkleBlock.header.GetHash()); - if (!pnetMan->getChainActive()->mapBlockIndex.count(merkleBlock.header.GetHash()) || - !pnetMan->getChainActive()->chainActive.Contains( - pnetMan->getChainActive()->mapBlockIndex[merkleBlock.header.GetHash()])) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); + { + LOCK(cs_main); + if (!pindex || !pnetMan->getChainActive()->chainActive.Contains(pindex)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); + } for (auto const &hash : vMatch) res.push_back(hash.GetHex()); @@ -676,7 +678,7 @@ UniValue signrawtransaction(const UniValue ¶ms, bool fHelp) CCoinsView viewDummy; CCoinsViewCache view(&viewDummy); { - LOCK(mempool.cs); + READLOCK(mempool.cs); CCoinsViewCache &viewChain = *pnetMan->getChainActive()->pcoinsTip; CCoinsViewMemPool viewMempool(&viewChain, mempool); view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 64f53440..e2979acc 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -80,6 +80,7 @@ void WalletTxToJSON(const CWalletTx &wtx, UniValue &entry) } if (confirms > 0) { + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); entry.push_back(Pair("blockhash", wtx.hashBlock.GetHex())); entry.push_back(Pair("blockindex", wtx.nIndex)); entry.push_back(Pair("blocktime", pnetMan->getChainActive()->mapBlockIndex[wtx.hashBlock]->GetBlockTime())); @@ -1205,9 +1206,7 @@ UniValue listsinceblock(const UniValue ¶ms, bool fHelp) uint256 blockId; blockId.SetHex(params[0].get_str()); - BlockMap::iterator it = pnetMan->getChainActive()->mapBlockIndex.find(blockId); - if (it != pnetMan->getChainActive()->mapBlockIndex.end()) - pindex = it->second; + pindex = pnetMan->getChainActive()->LookupBlockIndex(blockId); } if (params.size() > 1) diff --git a/src/sync.cpp b/src/sync.cpp index 681ceaaf..8f8d3ab2 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -23,14 +23,11 @@ #include "util/util.h" #include "util/utilstrencodings.h" -#include -#include -#include #include - +#include #ifdef DEBUG_LOCKCONTENTION -void PrintLockContention(const char *pszName, const char *pszFile, int nLine) +void PrintLockContention(const char *pszName, const char *pszFile, unsigned int nLine) { LogPrintf("LOCKCONTENTION: %s\n", pszName); LogPrintf("Locker: %s:%d\n", pszFile, nLine); @@ -38,6 +35,23 @@ void PrintLockContention(const char *pszName, const char *pszFile, int nLine) #endif /* DEBUG_LOCKCONTENTION */ #ifdef DEBUG_LOCKORDER +#include + +#ifdef __linux__ +uint64_t getTid(void) +{ + // "native" thread id used so the number correlates with what is shown in gdb + pid_t tid = (pid_t)syscall(SYS_gettid); + return tid; +} +#else +uint64_t getTid(void) +{ + uint64_t tid = boost::lexical_cast(boost::this_thread::get_id()); + return tid; +} +#endif + // // Early deadlock detection. // Problem being solved: @@ -64,10 +78,9 @@ struct CLockLocation return mutexName + " " + sourceFile + ":" + itostr(sourceLine) + (fTry ? " (TRY)" : ""); } - std::string MutexName() const { return mutexName; } - bool fTry; - + bool GetTry() const { return fTry; } private: + bool fTry; std::string mutexName; std::string sourceFile; int sourceLine; @@ -91,7 +104,7 @@ struct LockData std::mutex dd_mutex; } static lockdata; -static thread_local LockStack g_lockstack; +static thread_local std::unique_ptr lockstack; static void potential_deadlock_detected(const std::pair &mismatch, const LockStack &s1, @@ -109,19 +122,19 @@ static void potential_deadlock_detected(const std::pair &mismatc LogPrintf("POTENTIAL DEADLOCK DETECTED\n"); LogPrintf("Previous lock order was:\n"); - for (auto const &i : s2) + for (const PAIRTYPE(void *, CLockLocation) & i : s2) { if (i.first == mismatch.first) { LogPrintf(" (1)"); - if (!firstLocked && secondLocked && i.second.fTry) + if (!firstLocked && secondLocked && i.second.GetTry()) onlyMaybeDeadlock = true; firstLocked = true; } if (i.first == mismatch.second) { LogPrintf(" (2)"); - if (!secondLocked && firstLocked && i.second.fTry) + if (!secondLocked && firstLocked && i.second.GetTry()) onlyMaybeDeadlock = true; secondLocked = true; } @@ -130,19 +143,19 @@ static void potential_deadlock_detected(const std::pair &mismatc firstLocked = false; secondLocked = false; LogPrintf("Current lock order is:\n"); - for (auto const &i : s1) + for (const PAIRTYPE(void *, CLockLocation) & i : s1) { if (i.first == mismatch.first) { LogPrintf(" (1)"); - if (!firstLocked && secondLocked && i.second.fTry) + if (!firstLocked && secondLocked && i.second.GetTry()) onlyMaybeDeadlock = true; firstLocked = true; } if (i.first == mismatch.second) { LogPrintf(" (2)"); - if (!secondLocked && firstLocked && i.second.fTry) + if (!secondLocked && firstLocked && i.second.GetTry()) onlyMaybeDeadlock = true; secondLocked = true; } @@ -151,53 +164,55 @@ static void potential_deadlock_detected(const std::pair &mismatc assert(onlyMaybeDeadlock); } -static void push_lock(void *c, const CLockLocation &locklocation) +static void push_lock(void *c, const CLockLocation &locklocation, bool fTry) { - std::lock_guard lock(lockdata.dd_mutex); + if (lockstack.get() == NULL) + lockstack.reset(new LockStack); - g_lockstack.push_back(std::make_pair(c, locklocation)); + std::lock_guard lock(lockdata.dd_mutex); - for (const std::pair &i : g_lockstack) + (*lockstack).push_back(std::make_pair(c, locklocation)); + // If this is a blocking lock operation, we want to make sure that the locking order between 2 mutexes is consistent + // across the program + if (!fTry) { - if (i.first == c) - { - break; - } - - std::pair p1 = std::make_pair(i.first, c); - if (lockdata.lockorders.count(p1)) + for (const PAIRTYPE(void *, CLockLocation) & i : (*lockstack)) { - continue; - } - lockdata.lockorders[p1] = g_lockstack; + if (i.first == c) + break; - std::pair p2 = std::make_pair(c, i.first); - lockdata.invlockorders.insert(p2); - if (lockdata.lockorders.count(p2)) - { - potential_deadlock_detected(p1, lockdata.lockorders[p2], lockdata.lockorders[p1]); + std::pair p1 = std::make_pair(i.first, c); + // If this order has already been placed into the order map, we've already tested it + if (lockdata.lockorders.count(p1)) + continue; + lockdata.lockorders[p1] = (*lockstack); + // check to see if the opposite order has ever occurred, if so flag a possible deadlock + std::pair p2 = std::make_pair(c, i.first); + lockdata.invlockorders.insert(p2); + if (lockdata.lockorders.count(p2)) + potential_deadlock_detected(p1, lockdata.lockorders[p1], lockdata.lockorders[p2]); } } } -static void pop_lock() { g_lockstack.pop_back(); } -void EnterCritical(const char *pszName, const char *pszFile, int nLine, void *cs, bool fTry) +static void pop_lock() { (*lockstack).pop_back(); } +void EnterCritical(const char *pszName, const char *pszFile, unsigned int nLine, void *cs, bool fTry) { - push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry)); + push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry), fTry); } void LeaveCritical() { pop_lock(); } std::string LocksHeld() { std::string result; - for (const std::pair &i : g_lockstack) + for (const PAIRTYPE(void *, CLockLocation) & i : *lockstack) result += i.second.ToString() + std::string("\n"); return result; } -void AssertLockHeldInternal(const char *pszName, const char *pszFile, int nLine, void *cs) +void AssertLockHeldInternal(const char *pszName, const char *pszFile, unsigned int nLine, void *cs) { - for (const std::pair &i : g_lockstack) + for (const PAIRTYPE(void *, CLockLocation) & i : *lockstack) if (i.first == cs) return; fprintf(stderr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, @@ -205,9 +220,9 @@ void AssertLockHeldInternal(const char *pszName, const char *pszFile, int nLine, abort(); } -void AssertLockNotHeldInternal(const char *pszName, const char *pszFile, int nLine, void *cs) +void AssertLockNotHeldInternal(const char *pszName, const char *pszFile, unsigned int nLine, void *cs) { - for (const std::pair &i : g_lockstack) + for (const std::pair &i : *lockstack) { if (i.first == cs) { @@ -218,6 +233,154 @@ void AssertLockNotHeldInternal(const char *pszName, const char *pszFile, int nLi } } +void AssertWriteLockHeldInternal(const char *pszName, + const char *pszFile, + unsigned int nLine, + CSharedCriticalSection *cs) +{ + if (cs->try_lock()) // It would be better to check that this thread has the lock + { + fprintf(stderr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, + LocksHeld().c_str()); + fflush(stderr); + abort(); + } +} + +// BU normally CCriticalSection is a typedef, but when lockorder debugging is on we need to delete the critical +// section from the lockorder map +#ifdef DEBUG_LOCKORDER +CCriticalSection::CCriticalSection() : name(NULL) {} +CCriticalSection::CCriticalSection(const char *n) : name(n) +{ +// print the address of named critical sections so they can be found in the mutrace output +#ifdef ENABLE_MUTRACE + if (name) + { + LogPrintf("CCriticalSection %s at %p\n", name, this); + fflush(stdout); + } +#endif +} + +CCriticalSection::~CCriticalSection() +{ +#ifdef ENABLE_MUTRACE + if (name) + { + LogPrintf("Destructing %s\n", name); + fflush(stdout); + } +#endif + DeleteLock((void *)this); +} +#endif + +// BU normally CSharedCriticalSection is a typedef, but when lockorder debugging is on we need to delete the critical +// section from the lockorder map +#ifdef DEBUG_LOCKORDER +CSharedCriticalSection::CSharedCriticalSection() : name(NULL), exclusiveOwner(0) {} +CSharedCriticalSection::CSharedCriticalSection(const char *n) : name(n), exclusiveOwner(0) +{ +// print the address of named critical sections so they can be found in the mutrace output +#ifdef ENABLE_MUTRACE + if (name) + { + LogPrintf("CSharedCriticalSection %s at %p\n", name, this); + fflush(stdout); + } +#endif +} + +CSharedCriticalSection::~CSharedCriticalSection() +{ +#ifdef ENABLE_MUTRACE + if (name) + { + LogPrintf("Destructing CSharedCriticalSection %s\n", name); + fflush(stdout); + } +#endif + DeleteLock((void *)this); +} +#endif + + +void CSharedCriticalSection::lock_shared() +{ + uint64_t tid = getTid(); + // detect recursive locking + { + std::unique_lock lock(setlock); + assert(exclusiveOwner != tid); + auto alreadyLocked = sharedowners.find(tid); + if (alreadyLocked != sharedowners.end()) + { + LockInfo li = alreadyLocked->second; + LogPrintf("already locked at %s:%d\n", li.file, li.line); + assert(alreadyLocked == sharedowners.end()); + } + sharedowners[tid] = LockInfo("", 0); + } + boost::shared_mutex::lock_shared(); +} + +void CSharedCriticalSection::unlock_shared() +{ + // detect recursive locking + uint64_t tid = getTid(); + { + std::unique_lock lock(setlock); + auto alreadyLocked = sharedowners.find(tid); + if (alreadyLocked == sharedowners.end()) + { + LockInfo li = alreadyLocked->second; + LogPrintf("never locked at %s:%d\n", li.file, li.line); + assert(alreadyLocked != sharedowners.end()); + } + sharedowners.erase(tid); + } + boost::shared_mutex::unlock_shared(); +} + +bool CSharedCriticalSection::try_lock_shared() +{ + // detect recursive locking + uint64_t tid = getTid(); + std::unique_lock lock(setlock); + assert(exclusiveOwner != tid); + assert(sharedowners.find(tid) == sharedowners.end()); + + bool result = boost::shared_mutex::try_lock_shared(); + if (result) + { + sharedowners[tid] = LockInfo("", 0); + } + return result; +} +void CSharedCriticalSection::lock() +{ + boost::shared_mutex::lock(); + exclusiveOwner = getTid(); +} +void CSharedCriticalSection::unlock() +{ + uint64_t tid = getTid(); + assert(exclusiveOwner == tid); + exclusiveOwner = 0; + boost::shared_mutex::unlock(); +} + +bool CSharedCriticalSection::try_lock() +{ + bool result = boost::shared_mutex::try_lock(); + if (result) + { + exclusiveOwner = getTid(); + } + return result; +} + void DeleteLock(void *cs) { if (!lockdata.available) @@ -225,6 +388,7 @@ void DeleteLock(void *cs) // We're already shutting down. return; } + std::lock_guard lock(lockdata.dd_mutex); std::pair item = std::make_pair(cs, nullptr); LockOrders::iterator it = lockdata.lockorders.lower_bound(item); @@ -243,6 +407,4 @@ void DeleteLock(void *cs) } } -bool g_debug_lockorder_abort = true; - #endif /* DEBUG_LOCKORDER */ diff --git a/src/sync.h b/src/sync.h index 2bab96bc..c4cd8893 100644 --- a/src/sync.h +++ b/src/sync.h @@ -1,36 +1,25 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2009-2010 Satoshi Nakamoto - * Copyright (c) 2009-2016 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SYNC_H #define BITCOIN_SYNC_H #include "threadsafety.h" +#include "util/util.h" +#include "util/utiltime.h" #include #include #include #include +#include +#include +#include // for boost::thread_specific_ptr -#include #include -#include //////////////////////////////////////////////// // // @@ -82,7 +71,121 @@ class LOCKABLE AnnotatedMixin : public PARENT * Wrapped boost mutex: supports recursive locking, but no waiting * TODO: We should move away from using the recursive lock by default. */ +#ifndef DEBUG_LOCKORDER typedef AnnotatedMixin CCriticalSection; +#define CRITSEC(x) CCriticalSection x +#else // BU we need to remove the critical section from the lockorder map when destructed +class CCriticalSection : public AnnotatedMixin +{ +public: + const char *name; + CCriticalSection(const char *name); + CCriticalSection(); + ~CCriticalSection(); +}; +/** Define a critical section that is named in debug builds. + Named critical sections are useful in conjunction with a lock analyzer to discover bottlenecks. */ +#define CRITSEC(zzname) CCriticalSection zzname(#zzname) +#endif + +#ifndef DEBUG_LOCKORDER +typedef AnnotatedMixin CSharedCriticalSection; +/** Define a named, shared critical section that is named in debug builds. + Named critical sections are useful in conjunction with a lock analyzer to discover bottlenecks. */ +#define SCRITSEC(x) CSharedCriticalSection x +#else + +/** A shared critical section allows multiple entities to take the critical section in a "shared" mode, + but only one entity to take the critical section exclusively. + This is very useful for single-writer, many reader data structures. For example most of the containers + in the std and boost libraries follow these access semantics. + + A SharedCriticalSection is NOT recursive. +*/ +class CSharedCriticalSection : public AnnotatedMixin +{ +public: + class LockInfo + { + public: + const char *file; + unsigned int line; + LockInfo() : file(""), line(0) {} + LockInfo(const char *f, unsigned int l) : file(f), line(l) {} + }; + + std::mutex setlock; + std::map sharedowners; + const char *name; + uint64_t exclusiveOwner; + CSharedCriticalSection(const char *name); + CSharedCriticalSection(); + ~CSharedCriticalSection(); + void lock_shared(); + bool try_lock_shared(); + void unlock_shared(); + void lock(); + void unlock(); + bool try_lock(); +}; +#define SCRITSEC(zzname) CSharedCriticalSection zzname(#zzname) +#endif + + +// This object can be locked or shared locked some time during its lifetime. +// Subsequent locks or shared lock calls will be ignored. +// When it is deleted, the lock is released. +class CDeferredSharedLocker +{ + enum class LockState + { + UNLOCKED, + SHARED, + EXCLUSIVE + }; + CSharedCriticalSection &scs; + LockState state; + +public: + CDeferredSharedLocker(CSharedCriticalSection &scsp) : scs(scsp), state(LockState::UNLOCKED) {} + void lock_shared() + { + if (state == LockState::UNLOCKED) + { + scs.lock_shared(); + state = LockState::SHARED; + } + } + void lock() + { + if (state == LockState::UNLOCKED) + { + scs.lock(); + state = LockState::EXCLUSIVE; + } + } + + void unlock() + { + if (state == LockState::SHARED) + scs.unlock_shared(); + else if (state == LockState::EXCLUSIVE) + scs.unlock(); + state = LockState::UNLOCKED; + } + ~CDeferredSharedLocker() { unlock(); } +}; + +// This class unlocks a shared lock for the duration of its life +class CSharedUnlocker +{ + CSharedCriticalSection &cs; + +public: + CSharedUnlocker(CSharedCriticalSection &c) : cs(c) { cs.unlock_shared(); } + ~CSharedUnlocker() { cs.lock_shared(); } +}; + /** Wrapped boost mutex: supports waiting but not recursive locking */ typedef AnnotatedMixin CWaitableCriticalSection; @@ -90,31 +193,74 @@ typedef AnnotatedMixin CWaitableCriticalSection; /** Just a typedef for boost::condition_variable, can be wrapped later if desired */ typedef boost::condition_variable CConditionVariable; +/** Just a typedef for boost::condition_variable_any, can be wrapped later if desired -- c++11 version missing on win */ +typedef boost::condition_variable_any CCond; + #ifdef DEBUG_LOCKORDER -void EnterCritical(const char *pszName, const char *pszFile, int nLine, void *cs, bool fTry = false); +void EnterCritical(const char *pszName, const char *pszFile, unsigned int nLine, void *cs, bool fTry = false); void LeaveCritical(); +void DeleteLock(void *cs); std::string LocksHeld(); -void AssertLockHeldInternal(const char *pszName, const char *pszFile, int nLine, void *cs); +/** Asserts in debug builds if a critical section is not held. */ +void AssertLockHeldInternal(const char *pszName, const char *pszFile, unsigned int nLine, void *cs); +void AssertLockNotHeldInternal(const char *pszName, const char *pszFile, unsigned int nLine, void *cs); +/** Asserts in debug builds if a shared critical section is not exclusively held. */ +void AssertWriteLockHeldInternal(const char *pszName, + const char *pszFile, + unsigned int nLine, + CSharedCriticalSection *cs); #else -void static inline EnterCritical(const char *pszName, const char *pszFile, int nLine, void *cs, bool fTry = false) {} +void static inline EnterCritical(const char *pszName, + const char *pszFile, + unsigned int nLine, + void *cs, + bool fTry = false) +{ +} void static inline LeaveCritical() {} -void static inline AssertLockHeldInternal(const char *pszName, const char *pszFile, int nLine, void *cs) {} +void static inline AssertLockHeldInternal(const char *pszName, const char *pszFile, unsigned int nLine, void *cs) {} +void static inline AssertLockNotHeldInternal(const char *pszName, const char *pszFile, unsigned int nLine, void *cs) {} +void static inline AssertWriteLockHeldInternal(const char *pszName, + const char *pszFile, + unsigned int nLine, + CSharedCriticalSection *cs) +{ +} +static inline void DeleteLock(void *cs) {} #endif #define AssertLockHeld(cs) AssertLockHeldInternal(#cs, __FILE__, __LINE__, &cs) +#define AssertLockNotHeld(cs) AssertLockNotHeldInternal(#cs, __FILE__, __LINE__, &cs) +#define AssertWriteLockHeld(cs) AssertWriteLockHeldInternal(#cs, __FILE__, __LINE__, &cs) #ifdef DEBUG_LOCKCONTENTION -void PrintLockContention(const char *pszName, const char *pszFile, int nLine); +void PrintLockContention(const char *pszName, const char *pszFile, unsigned int nLine); #endif +#define LOCK_WARN_TIME (500ULL * 1000ULL * 1000ULL) + /** Wrapper around boost::unique_lock */ template class SCOPED_LOCKABLE CMutexLock { private: boost::unique_lock lock; +// Checking elapsed lock time is very inefficient compared to the lock/unlock operation so we must be able to +// turn the feature on and off at compile time. +#ifdef DEBUG_LOCKTIME + uint64_t lockedTime = 0; +#endif + const char *name = "unknown-name"; + const char *file = "unknown-file"; + unsigned int line = 0; - void Enter(const char *pszName, const char *pszFile, int nLine) + void Enter(const char *pszName, const char *pszFile, unsigned int nLine) { +#ifdef DEBUG_LOCKTIME + uint64_t startWait = GetStopwatch(); +#endif + name = pszName; + file = pszFile; + line = nLine; EnterCritical(pszName, pszFile, nLine, (void *)(lock.mutex())); #ifdef DEBUG_LOCKCONTENTION if (!lock.try_lock()) @@ -125,19 +271,39 @@ class SCOPED_LOCKABLE CMutexLock #ifdef DEBUG_LOCKCONTENTION } #endif + +#ifdef DEBUG_LOCKTIME + lockedTime = GetStopwatch(); + if (lockedTime - startWait > LOCK_WARN_TIME) + { + LOG(LCK, "Lock %s at %s:%d waited for %d ms\n", pszName, pszFile, nLine, (lockedTime - startWait)); + } +#endif } - bool TryEnter(const char *pszName, const char *pszFile, int nLine) + bool TryEnter(const char *pszName, const char *pszFile, unsigned int nLine) { + name = pszName; + file = pszFile; + line = nLine; EnterCritical(pszName, pszFile, nLine, (void *)(lock.mutex()), true); lock.try_lock(); if (!lock.owns_lock()) + { +#ifdef DEBUG_LOCKTIME + lockedTime = 0; +#endif LeaveCritical(); + } +#ifdef DEBUG_LOCKTIME + else + lockedTime = GetStopwatch(); +#endif return lock.owns_lock(); } public: - CMutexLock(Mutex &mutexIn, const char *pszName, const char *pszFile, int nLine, bool fTry = false) + CMutexLock(Mutex &mutexIn, const char *pszName, const char *pszFile, unsigned int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : lock(mutexIn, boost::defer_lock) { @@ -147,7 +313,7 @@ class SCOPED_LOCKABLE CMutexLock Enter(pszName, pszFile, nLine); } - CMutexLock(Mutex *pmutexIn, const char *pszName, const char *pszFile, int nLine, bool fTry = false) + CMutexLock(Mutex *pmutexIn, const char *pszName, const char *pszFile, unsigned int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) { if (!pmutexIn) @@ -163,17 +329,138 @@ class SCOPED_LOCKABLE CMutexLock ~CMutexLock() UNLOCK_FUNCTION() { if (lock.owns_lock()) + { LeaveCritical(); +#ifdef DEBUG_LOCKTIME + uint64_t doneTime = GetStopwatch(); + if (doneTime - lockedTime > LOCK_WARN_TIME) + { + LOG(LCK, "Lock %s at %s:%d remained locked for %d ms\n", name, file, line, doneTime - lockedTime); + } +#endif + } } operator bool() { return lock.owns_lock(); } }; +/** Wrapper around boost::unique_lock */ +template +class SCOPED_LOCKABLE CMutexReadLock +{ +private: + boost::shared_lock lock; + uint64_t lockedTime = 0; + const char *name = "unknown-name"; + const char *file = "unknown-file"; + unsigned int line = 0; + + void Enter(const char *pszName, const char *pszFile, unsigned int nLine) + { +#ifdef DEBUG_LOCKTIME + uint64_t startWait = GetStopwatch(); +#endif + name = pszName; + file = pszFile; + line = nLine; + EnterCritical(pszName, pszFile, nLine, (void *)(lock.mutex())); +// LOG(LCK,"try ReadLock %p %s by %d\n", lock.mutex(), name ? name : "", boost::this_thread::get_id()); +#ifdef DEBUG_LOCKCONTENTION + if (!lock.try_lock()) + { + PrintLockContention(pszName, pszFile, nLine); +#endif + lock.lock(); +#ifdef DEBUG_LOCKCONTENTION + } +#endif +// LOG(LCK,"ReadLock %p %s taken by %d\n", lock.mutex(), name ? name : "", boost::this_thread::get_id()); +#ifdef DEBUG_LOCKTIME + lockedTime = GetStopwatch(); + if (lockedTime - startWait > LOCK_WARN_TIME) + { + LOG(LCK, "Lock %s at %s:%d waited for %d ms\n", pszName, pszFile, nLine, (lockedTime - startWait)); + } +#endif + } + + bool TryEnter(const char *pszName, const char *pszFile, unsigned int nLine) + { + name = pszName; + file = pszFile; + line = nLine; + EnterCritical(pszName, pszFile, nLine, (void *)(lock.mutex()), true); + if (!lock.try_lock()) + { +#ifdef DEBUG_LOCKTIME + lockedTime = 0; +#endif + LeaveCritical(); + } +#ifdef DEBUG_LOCKTIME + else + lockedTime = GetStopwatch(); +#endif + return lock.owns_lock(); + } + +public: + CMutexReadLock(Mutex &mutexIn, const char *pszName, const char *pszFile, unsigned int nLine, bool fTry = false) + SHARED_LOCK_FUNCTION(mutexIn) + : lock(mutexIn, boost::defer_lock) + { + if (fTry) + TryEnter(pszName, pszFile, nLine); + else + Enter(pszName, pszFile, nLine); + } + + CMutexReadLock(Mutex *pmutexIn, const char *pszName, const char *pszFile, unsigned int nLine, bool fTry = false) + SHARED_LOCK_FUNCTION(pmutexIn) + { + if (!pmutexIn) + return; + + lock = boost::shared_lock(*pmutexIn, boost::defer_lock); + if (fTry) + TryEnter(pszName, pszFile, nLine); + else + Enter(pszName, pszFile, nLine); + } + + ~CMutexReadLock() UNLOCK_FUNCTION() + { + if (lock.owns_lock()) + { + LeaveCritical(); +#ifdef DEBUG_LOCKTIME + int64_t doneTime = GetStopwatch(); + if (doneTime - lockedTime > LOCK_WARN_TIME) + { + LOG(LCK, "Lock %s at %s:%d remained locked for %d ms\n", name, file, line, doneTime - lockedTime); + } +#endif + } + // When lock is destructed it will release + } + + operator bool() { return lock.owns_lock(); } +}; + +typedef CMutexReadLock CReadBlock; +typedef CMutexLock CWriteBlock; typedef CMutexLock CCriticalBlock; -#define LOCK(cs) CCriticalBlock criticalblock(cs, #cs, __FILE__, __LINE__) -#define LOCK2(cs1, cs2) \ - CCriticalBlock criticalblock1(cs1, #cs1, __FILE__, __LINE__), criticalblock2(cs2, #cs2, __FILE__, __LINE__) +#define READLOCK(cs) CReadBlock UNIQUIFY(readblock)(cs, #cs, __FILE__, __LINE__) +#define WRITELOCK(cs) CWriteBlock UNIQUIFY(writeblock)(cs, #cs, __FILE__, __LINE__) +#define READLOCK2(cs1, cs2) \ + CReadBlock UNIQUIFY(readblock1)(cs1, #cs1, __FILE__, __LINE__), UNIQUIFY(readblock2)(cs2, #cs2, __FILE__, __LINE__) +#define TRY_READ_LOCK(cs, name) CReadBlock name(cs, #cs, __FILE__, __LINE__, true) + +#define LOCK(cs) CCriticalBlock UNIQUIFY(criticalblock)(cs, #cs, __FILE__, __LINE__) +#define LOCK2(cs1, cs2) \ + CCriticalBlock UNIQUIFY(criticalblock1)(cs1, #cs1, __FILE__, __LINE__), \ + UNIQUIFY(criticalblock2)(cs2, #cs2, __FILE__, __LINE__) #define TRY_LOCK(cs, name) CCriticalBlock name(cs, #cs, __FILE__, __LINE__, true) #define ENTER_CRITICAL_SECTION(cs) \ @@ -191,22 +478,25 @@ typedef CMutexLock CCriticalBlock; class CSemaphore { private: - std::condition_variable condition; - std::mutex mutex; + boost::condition_variable condition; + boost::mutex mutex; int value; public: - explicit CSemaphore(int init) : value(init) {} + CSemaphore(int init) : value(init) {} void wait() { - std::unique_lock lock(mutex); - condition.wait(lock, [&]() { return value >= 1; }); + boost::unique_lock lock(mutex); + while (value < 1) + { + condition.wait(lock); + } value--; } bool try_wait() { - std::lock_guard lock(mutex); + boost::unique_lock lock(mutex); if (value < 1) return false; value--; @@ -216,14 +506,13 @@ class CSemaphore void post() { { - std::lock_guard lock(mutex); + boost::unique_lock lock(mutex); value++; } condition.notify_one(); } }; - /** RAII-style semaphore lock */ class CSemaphoreGrant { @@ -265,7 +554,7 @@ class CSemaphoreGrant } CSemaphoreGrant() : sem(NULL), fHaveGrant(false) {} - explicit CSemaphoreGrant(CSemaphore &sema, bool fTry = false) : sem(&sema), fHaveGrant(false) + CSemaphoreGrant(CSemaphore &sema, bool fTry = false) : sem(&sema), fHaveGrant(false) { if (fTry) TryAcquire(); diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 62a3c62b..8ee355c2 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -171,6 +171,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) } else { + LOCK(stack.back()->cs_utxo); const Coin &entry = stack.back()->AccessCoin(COutPoint(txid, 0)); BOOST_CHECK(coin == entry); } @@ -225,10 +226,15 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) for (auto it = result.begin(); it != result.end(); it++) { bool have = stack.back()->HaveCoin(it->first); - const Coin &coin = stack.back()->AccessCoin(it->first); - BOOST_CHECK(have == !coin.IsSpent()); - BOOST_CHECK(coin == it->second); - if (coin.IsSpent()) + bool isspent = true; + { + LOCK(stack.back()->cs_utxo); + const Coin &coin = stack.back()->AccessCoin(it->first); + isspent = coin.IsSpent(); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); + } + if (isspent) { missed_an_entry = true; } diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index ff35a8a6..fae8a0ea 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -59,7 +59,7 @@ CBlockIndex CreateBlockIndex(int nHeight) bool TestSequenceLocks(const CTransaction &tx, int flags) { - LOCK(mempool.cs); + READLOCK(mempool.cs); return CheckSequenceLocks(tx, flags); } diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 75dc86a4..c51c2f1b 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -13,7 +13,6 @@ #include "consensus/validation.h" #include "crypto/sha256.h" #include "fs.h" -#include "init.h" #include "key.h" #include "main.h" #include "net/messages.h" @@ -79,7 +78,7 @@ TestingSetup::~TestingSetup() fs::remove_all(pathTemp); } -TestChain100Setup::TestChain100Setup() : TestingSetup("TESTNET0-TEMPORARY") +TestChain100Setup::TestChain100Setup() : TestingSetup("REGTEST") { // Generate a 100-block chain: coinbaseKey.MakeNewKey(true); @@ -99,8 +98,8 @@ TestChain100Setup::TestChain100Setup() : TestingSetup("TESTNET0-TEMPORARY") CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector &txns, const CScript &scriptPubKey) { std::unique_ptr pblocktemplate(CreateNewBlock(pwallet, scriptPubKey, false)); - std::shared_ptr pblock = std::make_shared(); - *pblock = pblocktemplate->block; + CBlock block = pblocktemplate->block; + CBlock *pblock = █ // Replace mempool-selected txns with just coinbase plus passed-in txns: pblock->vtx.resize(1); @@ -108,7 +107,7 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vectorvtx.push_back(tx); // IncrementExtraNonce creates a valid coinbase and merkleRoot unsigned int extraNonce = 0; - IncrementExtraNonce(pblock.get(), pnetMan->getChainActive()->chainActive.Tip(), extraNonce); + IncrementExtraNonce(pblock, pnetMan->getChainActive()->chainActive.Tip(), extraNonce); while (!CheckProofOfWork(pblock->GetHash(), pblock->nBits, pnetMan->getActivePaymentNetwork()->GetConsensus())) ++pblock->nNonce; @@ -133,14 +132,14 @@ CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(CTransaction &tx, CTxMemPool *poo inChainValue, spendsCoinbase, sigOpCount, lp); } -void ShutdownTest(void *parg) { exit(0); } -void StartShutdownTest() { exit(0); } -bool ShutdownRequestedTest() { return false; } +void Shutdown(void *parg) { exit(0); } +// void StartShutdown() { exit(0); } +// bool ShutdownRequested() { return false; } using namespace boost::program_options; struct StartupShutdown { - void StartupShutdownTest() + StartupShutdown() { options_description optDef("Options"); optDef.add_options()("testhelp", "program options information")( @@ -172,7 +171,7 @@ struct StartupShutdown } } } - // ~StartupShutdownTest() { } + ~StartupShutdown() {} }; BOOST_GLOBAL_FIXTURE(StartupShutdown); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 3098a21f..be0c390d 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -179,7 +179,7 @@ bool CTxMemPool::UpdateForDescendants(txiter updateIt, // add fee/size information for such descendants to the parent. void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashesToUpdate) { - LOCK(cs); + WRITELOCK(cs); // For each entry in vHashesToUpdate, store the set of in-mempool, but not // in-vHashesToUpdate transactions, so that we don't have to recalculate // descendants when we come across a previously seen entry. @@ -236,7 +236,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, std::string &errString, bool fSearchForParents /* = true */) { - LOCK(cs); + READLOCK(cs); setEntries setAncestors; return _CalculateMemPoolAncestors(entry, setAncestors, limitAncestorCount, limitAncestorSize, limitDescendantCount, limitDescendantSize, errString, fSearchForParents); @@ -531,13 +531,13 @@ bool CTxMemPool::isSpent(const COutPoint &outpoint) unsigned int CTxMemPool::GetTransactionsUpdated() const { - LOCK(cs); + READLOCK(cs); return nTransactionsUpdated; } void CTxMemPool::AddTransactionsUpdated(unsigned int n) { - LOCK(cs); + WRITELOCK(cs); nTransactionsUpdated += n; } @@ -661,7 +661,7 @@ void CTxMemPool::_CalculateDescendants(txiter entryit, setEntries &setDescendant void CTxMemPool::remove(const CTransaction &origTx, std::list &removed, bool fRecursive) { - LOCK(cs); + WRITELOCK(cs); _remove(origTx, removed, fRecursive); } @@ -713,7 +713,7 @@ void CTxMemPool::_remove(const CTransaction &origTx, std::list void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags) { // Remove transactions spending a coinbase which are now immature and no-longer-final transactions - LOCK(cs); + WRITELOCK(cs); list transactionsToRemove; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { @@ -758,7 +758,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem void CTxMemPool::removeConflicts(const CTransaction &tx, std::list &removed) { - LOCK(cs); + WRITELOCK(cs); _removeConflicts(tx, removed); } @@ -789,7 +789,7 @@ void CTxMemPool::removeForBlock(const std::vector &vtx, std::list &conflicts, bool fCurrentEstimate) { - LOCK(cs); + WRITELOCK(cs); std::vector entries; for (const auto &tx : vtx) { @@ -827,7 +827,7 @@ void CTxMemPool::_clear() void CTxMemPool::clear() { - LOCK(cs); + WRITELOCK(cs); _clear(); } @@ -842,7 +842,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const uint64_t checkTotal = 0; uint64_t innerUsage = 0; - LOCK(cs); + READLOCK(cs); // LogPrintf("MEMPOOL", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), // (unsigned int)mapNextTx.size()); @@ -957,7 +957,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const void CTxMemPool::queryHashes(vector &vtxid) const { - LOCK(mempool.cs); + READLOCK(mempool.cs); _queryHashes(vtxid); } void CTxMemPool::_queryHashes(vector &vtxid) const @@ -983,14 +983,14 @@ bool CTxMemPool::_lookup(const uint256 &hash, CTxMemPoolEntry &result) const bool CTxMemPool::lookup(const uint256 &hash, CTxMemPoolEntry &result) const { - LOCK(cs); + READLOCK(cs); return _lookup(hash, result); } bool CTxMemPool::lookup(const uint256 &hash, CTransaction &result) const { - LOCK(cs); + READLOCK(cs); return _lookup(hash, result); } @@ -1006,12 +1006,12 @@ bool CTxMemPool::_lookup(const uint256 &hash, CTransaction &result) const CFeeRate CTxMemPool::estimateFee(int nBlocks) const { - LOCK(cs); + READLOCK(cs); return minerPolicyEstimator->estimateFee(nBlocks); } CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const { - LOCK(cs); + READLOCK(cs); return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); } @@ -1019,7 +1019,7 @@ bool CTxMemPool::WriteFeeEstimates(CAutoFile &fileout) const { try { - LOCK(cs); + READLOCK(cs); fileout << 109900; // version required to read: 0.10.99 or later fileout << CLIENT_VERSION; // version that wrote the file minerPolicyEstimator->Write(fileout); @@ -1041,7 +1041,7 @@ bool CTxMemPool::ReadFeeEstimates(CAutoFile &filein) if (nVersionRequired > CLIENT_VERSION) return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - LOCK(cs); + WRITELOCK(cs); minerPolicyEstimator->Read(filein); } catch (const std::exception &) @@ -1063,7 +1063,7 @@ CTransactionRef CTxMemPool::_get(const uint256 &hash) const CTransactionRef CTxMemPool::get(const uint256 &hash) const { - LOCK(cs); + READLOCK(cs); return _get(hash); } @@ -1073,7 +1073,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256 hash, const CAmount &nFeeDelta) { { - LOCK(cs); + WRITELOCK(cs); std::pair &deltas = mapDeltas[hash]; deltas.first += dPriorityDelta; deltas.second += nFeeDelta; @@ -1097,7 +1097,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256 hash, void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const { - LOCK(cs); + READLOCK(cs); _ApplyDeltas(hash, dPriorityDelta, nFeeDelta); } @@ -1114,7 +1114,7 @@ void CTxMemPool::_ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmoun void CTxMemPool::ClearPrioritisation(const uint256 hash) { - LOCK(cs); + WRITELOCK(cs); mapDeltas.erase(hash); } void CTxMemPool::_ClearPrioritisation(const uint256 hash) { mapDeltas.erase(hash); } @@ -1159,7 +1159,7 @@ bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const size_t CTxMemPool::DynamicMemoryUsage() const { - LOCK(cs); + READLOCK(cs); // Estimate the overhead of mapTx to be 12 pointers + an allocation, as no exact formula for // boost::multi_index_contained is implemented. return _DynamicMemoryUsage(); @@ -1187,7 +1187,7 @@ void CTxMemPool::_RemoveStaged(setEntries &stage) int CTxMemPool::Expire(int64_t time, std::vector &vCoinsToUncache) { - LOCK(cs); + WRITELOCK(cs); indexed_transaction_set::index::type::iterator it = mapTx.get().begin(); setEntries toremove; while (it != mapTx.get().end() && it->GetTime() < time) @@ -1209,7 +1209,7 @@ int CTxMemPool::Expire(int64_t time, std::vector &vCoinsToUncache) bool CTxMemPool::addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate) { NotifyEntryAdded(entry.GetSharedTx()); - LOCK(cs); + WRITELOCK(cs); setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); std::string dummy; @@ -1265,7 +1265,7 @@ const CTxMemPool::setEntries &CTxMemPool::GetMemPoolChildren(txiter entry) const CFeeRate CTxMemPool::GetMinFee(size_t sizelimit) const { - LOCK(cs); + READLOCK(cs); return _GetMinFee(sizelimit); } @@ -1309,7 +1309,7 @@ void CTxMemPool::trackPackageRemoved(const CFeeRate &rate) void CTxMemPool::TrimToSize(size_t sizelimit, std::vector *pvNoSpendsRemaining) { - LOCK(cs); + WRITELOCK(cs); unsigned nTxnRemoved = 0; CFeeRate maxFeeRateRemoved(0); while (_DynamicMemoryUsage() > sizelimit) @@ -1361,8 +1361,7 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector *pvNoSpends void CTxMemPool::UpdateTransactionsPerSecond() { - boost::mutex::scoped_lock lock(cs_txPerSec); - + std::lock_guard lock(cs_txPerSec); static int64_t nLastTime = GetTime(); double nSecondsToAverage = 60; // Length of time in seconds to smooth the tx rate over int64_t nNow = GetTime(); diff --git a/src/txmempool.h b/src/txmempool.h index 6ae13832..27e2f1a6 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -463,7 +463,7 @@ class CTxMemPool void trackPackageRemoved(const CFeeRate &rate); - boost::mutex cs_txPerSec; + std::mutex cs_txPerSec; double nTxPerSec; // BU: tx's per second accepted into the mempool public: @@ -488,7 +488,7 @@ class CTxMemPool CompareTxMemPoolEntryByScore> > > indexed_transaction_set; - mutable CCriticalSection cs; + mutable CSharedCriticalSection cs; indexed_transaction_set mapTx; typedef indexed_transaction_set::nth_index<0>::type::iterator txiter; struct CompareIteratorByHash @@ -657,31 +657,31 @@ class CTxMemPool unsigned long size() { - LOCK(cs); + READLOCK(cs); return mapTx.size(); } unsigned long _size() { return mapTx.size(); } uint64_t GetTotalTxSize() { - LOCK(cs); + READLOCK(cs); return totalTxSize; } bool exists(uint256 hash) const { - LOCK(cs); + READLOCK(cs); return (mapTx.count(hash) != 0); } bool _exists(uint256 hash) const { return (mapTx.count(hash) != 0); } double TransactionsPerSecond() { - boost::mutex::scoped_lock lock(cs_txPerSec); + std::lock_guard lock(cs_txPerSec); return nTxPerSec; } bool exists(const COutPoint &outpoint) const { - LOCK(cs); + READLOCK(cs); auto it = mapTx.find(outpoint.hash); return (it != mapTx.end() && outpoint.n < it->GetTx().vout.size()); } diff --git a/src/util/util.cpp b/src/util/util.cpp index 46b4aae1..548dd5d0 100644 --- a/src/util/util.cpp +++ b/src/util/util.cpp @@ -90,7 +90,7 @@ #include // for startswith() and endswith() #include #include -#include +#include // for boost::thread_specific_ptr #include #include diff --git a/src/util/util.h b/src/util/util.h index 672a5a16..03f95089 100644 --- a/src/util/util.h +++ b/src/util/util.h @@ -41,6 +41,12 @@ #include #include +#define UNIQUE2(pfx, LINE) pfx##LINE +#define UNIQUE1(pfx, LINE) UNIQUE2(pfx, LINE) +/// UNIQUIFY is a macro that appends the current file's line number to the passed prefix, creating a symbol +// that is unique in this file. +#define UNIQUIFY(pfx) UNIQUE1(pfx, __LINE__) + static const bool DEFAULT_LOGTIMEMICROS = false; static const bool DEFAULT_LOGIPS = false; static const bool DEFAULT_LOGTIMESTAMPS = true; diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 07d962df..e4f8da9d 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -26,10 +26,7 @@ CMainSignals &GetMainSignals() { return g_signals; } void RegisterValidationInterface(CValidationInterface *pwalletIn) { g_signals.UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); - g_signals.TransactionAddedToMempool.connect( - boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); - g_signals.BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); - g_signals.BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); @@ -45,10 +42,7 @@ void UnregisterValidationInterface(CValidationInterface *pwalletIn) g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.TransactionAddedToMempool.disconnect( - boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); - g_signals.BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); - g_signals.BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); g_signals.UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); g_signals.NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } @@ -60,9 +54,12 @@ void UnregisterAllValidationInterfaces() g_signals.Broadcast.disconnect_all_slots(); g_signals.Inventory.disconnect_all_slots(); g_signals.SetBestChain.disconnect_all_slots(); - g_signals.TransactionAddedToMempool.disconnect_all_slots(); - g_signals.BlockConnected.disconnect_all_slots(); - g_signals.BlockDisconnected.disconnect_all_slots(); + g_signals.SyncTransaction.disconnect_all_slots(); g_signals.UpdatedBlockTip.disconnect_all_slots(); g_signals.NewPoWValidBlock.disconnect_all_slots(); } + +void SyncWithWallets(const CTransactionRef &ptx, const CBlock *pblock, int txIdx) +{ + g_signals.SyncTransaction(ptx, pblock, txIdx); +} diff --git a/src/validationinterface.h b/src/validationinterface.h index d83d02fd..1377f1bf 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -45,24 +45,20 @@ void RegisterValidationInterface(CValidationInterface *pwalletIn); void UnregisterValidationInterface(CValidationInterface *pwalletIn); /** Unregister all wallets from core */ void UnregisterAllValidationInterfaces(); +/** Push an updated transaction to all registered wallets, pass NULL if block not known, pass -1 if txIdx not known */ +void SyncWithWallets(const CTransactionRef &ptx, const CBlock *pblock, int txIdx); class CValidationInterface { protected: virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} - virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} - virtual void BlockConnected(const std::shared_ptr &block, - const CBlockIndex *pindex, - const std::vector &txnConflicted) - { - } - virtual void BlockDisconnected(const std::shared_ptr &block) {} + virtual void SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock, int txIdx) {} virtual void SetBestChain(const CBlockLocator &locator) {} virtual void Inventory(const uint256 &hash) {} virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman *connman) {} virtual void BlockChecked(const CBlock &, const CValidationState &) {} virtual void GetScriptForMining(boost::shared_ptr &){}; - virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr &block) {} + virtual void NewPoWValidBlock(const CBlockIndex *pindex, const CBlock *block) {} friend void ::RegisterValidationInterface(CValidationInterface *); friend void ::UnregisterValidationInterface(CValidationInterface *); friend void ::UnregisterAllValidationInterfaces(); @@ -72,17 +68,8 @@ struct CMainSignals { /** Notifies listeners of updated block chain tip */ boost::signals2::signal UpdatedBlockTip; - /** Notifies listeners of a transaction having been added to mempool. */ - boost::signals2::signal TransactionAddedToMempool; - /** - * Notifies listeners of a block being connected. - * Provides a vector of transactions evicted from the mempool as a result. - */ - boost::signals2::signal< - void(const std::shared_ptr &, const CBlockIndex *pindex, const std::vector &)> - BlockConnected; - /** Notifies listeners of a block being disconnected */ - boost::signals2::signal &)> BlockDisconnected; + /** Notifies listeners of updated transaction data (transaction, and optionally the block it is found in. */ + boost::signals2::signal SyncTransaction; /** Notifies listeners of a new active block chain. */ boost::signals2::signal SetBestChain; /** Notifies listeners about an inventory item being seen on the network. */ @@ -98,7 +85,7 @@ struct CMainSignals * has been received and connected to the headers tree, though not validated * yet. */ - boost::signals2::signal &)> NewPoWValidBlock; + boost::signals2::signal NewPoWValidBlock; }; CMainSignals &GetMainSignals(); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index dea21c85..83868436 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -683,6 +683,7 @@ bool CWallet::AddToWallet(const CWalletTx &wtxIn, bool fFromLoadWallet, CWalletD wtx.nTimeSmart = wtx.nTimeReceived; if (!wtxIn.hashUnset()) { + READLOCK(pnetMan->getChainActive()->cs_mapBlockIndex); if (pnetMan->getChainActive()->mapBlockIndex.count(wtxIn.hashBlock)) { int64_t latestNow = wtx.nTimeReceived; @@ -900,9 +901,9 @@ void CWallet::MarkConflicted(const uint256 &hashBlock, const uint256 &hashTx) LOCK2(cs_main, cs_wallet); int conflictconfirms = 0; - if (pnetMan->getChainActive()->mapBlockIndex.count(hashBlock)) + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hashBlock); + if (pindex) { - CBlockIndex *pindex = pnetMan->getChainActive()->mapBlockIndex[hashBlock]; if (pnetMan->getChainActive()->chainActive.Contains(pindex)) { conflictconfirms = -(pnetMan->getChainActive()->chainActive.Height() - pindex->nHeight + 1); @@ -960,12 +961,10 @@ void CWallet::MarkConflicted(const uint256 &hashBlock, const uint256 &hashTx) } } -void CWallet::SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock) +void CWallet::SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock, int txIdx) { LOCK2(cs_main, cs_wallet); - const CTransaction &tx = *ptx; - if (!AddToWalletIfInvolvingMe(ptx, pblock, true)) { return; // Not one of ours @@ -974,53 +973,13 @@ void CWallet::SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock) // If a transaction changes 'conflicted' state, that changes the balance // available of the outputs it spends. So force those to be // recomputed, also: - for (auto const &txin : tx.vin) + for (const CTxIn &txin : ptx->vin) { if (mapWallet.count(txin.prevout.hash)) mapWallet[txin.prevout.hash].MarkDirty(); } } -void CWallet::TransactionAddedToMempool(const CTransactionRef &ptx) -{ - LOCK2(cs_main, cs_wallet); - SyncTransaction(ptx); -} - -void CWallet::BlockConnected(const std::shared_ptr &pblock, - const CBlockIndex *pindex, - const std::vector &vtxConflicted) -{ - LOCK2(cs_main, cs_wallet); - // TODO: Tempoarily ensure that mempool removals are notified before - // connected transactions. This shouldn't matter, but the abandoned state of - // transactions in our wallet is currently cleared when we receive another - // notification and there is a race condition where notification of a - // connected conflict might cause an outside process to abandon a - // transaction and then have it inadvertantly cleared by the notification - // that the conflicted transaction was evicted. - - for (const CTransactionRef &ptx : vtxConflicted) - { - SyncTransaction(ptx); - } - for (size_t i = 0; i < pblock->vtx.size(); i++) - { - SyncTransaction(pblock->vtx[i], pblock.get()); - } -} - -void CWallet::BlockDisconnected(const std::shared_ptr &pblock) -{ - LOCK2(cs_main, cs_wallet); - - for (const CTransactionRef &ptx : pblock->vtx) - { - SyncTransaction(ptx); - } -} - - isminetype CWallet::IsMine(const CTxIn &txin) const { { @@ -1341,21 +1300,24 @@ void CWallet::ReacceptWalletTransactions() // If transactions aren't being broadcasted, don't let them into local mempool either if (!fBroadcastTransactions) return; - LOCK2(cs_main, cs_wallet); + std::map mapSorted; - // Sort pending wallet transactions based on their initial wallet insertion order - for (auto const &item : mapWallet) { - const uint256 &wtxid = item.first; - CWalletTx wtx = item.second; - assert(wtx.tx->GetHash() == wtxid); + LOCK2(cs_main, cs_wallet); + // Sort pending wallet transactions based on their initial wallet insertion order + for (auto const &item : mapWallet) + { + const uint256 &wtxid = item.first; + CWalletTx wtx = item.second; + assert(wtx.tx->GetHash() == wtxid); - int nDepth = wtx.GetDepthInMainChain(); + int nDepth = wtx.GetDepthInMainChain(); - if (!wtx.tx->IsCoinBase() && !wtx.tx->IsCoinStake() && (nDepth == 0 && !wtx.isAbandoned())) - { - mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); + if (!wtx.tx->IsCoinBase() && !wtx.tx->IsCoinStake() && (nDepth == 0 && !wtx.isAbandoned())) + { + mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); + } } } @@ -1363,9 +1325,8 @@ void CWallet::ReacceptWalletTransactions() for (auto const &item : mapSorted) { CWalletTx &wtx = *(item.second); - - LOCK(mempool.cs); wtx.AcceptToMemoryPool(false); + SyncWithWallets(wtx.tx, nullptr, -1); } } @@ -1556,7 +1517,6 @@ CAmount CWalletTx::GetChange() const bool CWalletTx::InMempool() const { - LOCK(mempool.cs); if (mempool.exists(tx->GetHash())) { return true; @@ -2477,6 +2437,17 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, */ bool CWallet::CommitTransaction(CWalletTx &wtxNew, CReserveKey &reservekey, CConnman *connman, CValidationState &state) { + if (fBroadcastTransactions) + { + // Broadcast + if (!wtxNew.AcceptToMemoryPool(false)) + { + // This must not fail. The transaction has already been signed and recorded. + LogPrintf("CommitTransaction(): Error: Transaction not valid\n"); + return false; + } + } + { LOCK2(cs_main, cs_wallet); LogPrintf("CommitTransaction:\n%s", wtxNew.tx->ToString()); @@ -2511,13 +2482,7 @@ bool CWallet::CommitTransaction(CWalletTx &wtxNew, CReserveKey &reservekey, CCon if (fBroadcastTransactions) { - // Broadcast - if (!wtxNew.AcceptToMemoryPool(false)) - { - // This must not fail. The transaction has already been signed and recorded. - LogPrintf("CommitTransaction(): Error: Transaction not valid\n"); - return false; - } + SyncWithWallets(wtxNew.tx, nullptr, -1); wtxNew.RelayWalletTransaction(connman); } } @@ -3144,12 +3109,11 @@ void CWallet::GetKeyBirthTimes(std::map &mapKeyBirth) const { // iterate over all wallet transactions... const CWalletTx &wtx = (*it).second; - BlockMap::const_iterator blit = pnetMan->getChainActive()->mapBlockIndex.find(wtx.hashBlock); - if (blit != pnetMan->getChainActive()->mapBlockIndex.end() && - pnetMan->getChainActive()->chainActive.Contains(blit->second)) + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(wtx.hashBlock); + if (pindex && pnetMan->getChainActive()->chainActive.Contains(pindex)) { // ... which are already in a block - int nHeight = blit->second->nHeight; + int nHeight = pindex->nHeight; for (auto const &txout : wtx.tx->vout) { // iterate over all their outputs @@ -3159,7 +3123,7 @@ void CWallet::GetKeyBirthTimes(std::map &mapKeyBirth) const // ... and all their affected keys std::map::iterator rit = mapKeyFirstBlock.find(keyid); if (rit != mapKeyFirstBlock.end() && nHeight < rit->second->nHeight) - rit->second = blit->second; + rit->second = pindex; } vAffected.clear(); } @@ -3250,12 +3214,7 @@ int CMerkleTx::SetMerkleBranch(const CBlock &block) } // Is the tx in a block that's in the main chain - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hashBlock); - if (mi == pnetMan->getChainActive()->mapBlockIndex.end()) - { - return 0; - } - const CBlockIndex *pindex = (*mi).second; + const CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hashBlock); if (!pindex || !pnetMan->getChainActive()->chainActive.Contains(pindex)) { return 0; @@ -3271,12 +3230,8 @@ int CMerkleTx::GetDepthInMainChain(const CBlockIndex *&pindexRet) const } AssertLockHeld(cs_main); // Find the block it claims to be in - BlockMap::iterator mi = pnetMan->getChainActive()->mapBlockIndex.find(hashBlock); - if (mi == pnetMan->getChainActive()->mapBlockIndex.end()) - { - return 0; - } - CBlockIndex *pindex = (*mi).second; + CBlockIndex *pindex = pnetMan->getChainActive()->LookupBlockIndex(hashBlock); + LOCK(cs_main); // for chainActive if (!pindex || !pnetMan->getChainActive()->chainActive.Contains(pindex)) { return 0; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c8456746..dd4938f2 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -684,7 +684,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface void MarkDirty(); bool AddToWallet(const CWalletTx &wtxIn, bool fFromLoadWallet, CWalletDB *pwalletdb); - void SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock = nullptr); + void SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock, int txIndex = -1); bool AddToWalletIfInvolvingMe(const CTransactionRef &ptx, const CBlock *pblock, bool fUpdate); int ScanForWalletTransactions(CBlockIndex *pindexStart, bool fUpdate = false); void ReacceptWalletTransactions(); @@ -766,11 +766,6 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface void SetBestChain(const CBlockLocator &loc); DBErrors LoadWallet(bool &fFirstRunRet); - void TransactionAddedToMempool(const CTransactionRef &tx) override; - void BlockConnected(const std::shared_ptr &pblock, - const CBlockIndex *pindex, - const std::vector &vtxConflicted) override; - void BlockDisconnected(const std::shared_ptr &pblock) override; DBErrors ZapWalletTx(std::vector &vWtx); From 3c8bc5a3f6e5b92ca8d1ce1dab24d1efe5a5239d Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Wed, 13 Feb 2019 23:50:19 -0500 Subject: [PATCH 07/46] Seperate logger for easier management. remove translation print wrapper (#130) * Seperate logger for easier management. remove translation print wrapper * rebase fixes * fix fPrintToConsole reference * fix formatting * init logger earlier in startup process * remove unused ui interfaces and replace them with logging messages * fix issue where category was passed into logprintf * fix invalid log prints --- src/.formatted-files | 3 - src/Makefile.am | 5 +- src/bench/bench_bitcoin.cpp | 2 +- src/blockgeneration/miner.cpp | 2 +- src/blockgeneration/minter.cpp | 2 +- src/chain/block.cpp | 3 - src/chain/tx.cpp | 16 +- src/coins.cpp | 10 +- src/core_read.cpp | 1 + src/dbwrapper.h | 1 + src/eccoind.cpp | 10 +- src/httprpc.cpp | 6 +- src/httpserver.cpp | 18 +- src/init.cpp | 415 ++++++++++++++++----------------- src/init.h | 2 + src/kernel.cpp | 63 ++--- src/keystore.cpp | 1 + src/main.cpp | 41 +--- src/net/addrman.h | 1 + src/net/messages.cpp | 6 +- src/net/net.cpp | 22 +- src/net/netbase.cpp | 1 + src/net/protocol.cpp | 1 + src/networks/netman.cpp | 4 +- src/noui.cpp | 65 ------ src/noui.h | 25 -- src/policy/fees.cpp | 1 + src/processblock.cpp | 9 +- src/random.cpp | 3 +- src/rpc/rpcdump.cpp | 2 +- src/rpc/rpcnet.cpp | 2 +- src/rpc/rpcprotocol.cpp | 1 + src/rpc/rpcserver.cpp | 6 +- src/sync.cpp | 1 + src/test/test_bitcoin.cpp | 18 +- src/timedata.cpp | 9 +- src/torcontrol.cpp | 2 +- src/ui_interface.h | 120 ---------- src/util/logger.cpp | 212 +++++++++++++++++ src/util/logger.h | 118 ++++++++++ src/util/util.cpp | 225 +----------------- src/util/util.h | 112 +-------- src/verifydb.cpp | 12 +- src/wallet/crypter.cpp | 1 + src/wallet/wallet.cpp | 118 ++++------ src/wallet/wallet.h | 22 +- 46 files changed, 692 insertions(+), 1028 deletions(-) delete mode 100644 src/noui.cpp delete mode 100644 src/noui.h delete mode 100644 src/ui_interface.h create mode 100644 src/util/logger.cpp create mode 100644 src/util/logger.h diff --git a/src/.formatted-files b/src/.formatted-files index 1071b7cb..edc31053 100644 --- a/src/.formatted-files +++ b/src/.formatted-files @@ -99,8 +99,6 @@ networks/netman.cpp networks/netman.h networks/network.h networks/networktemplate.h -noui.cpp -noui.h policy/fees.cpp policy/fees.h policy/policy.cpp @@ -171,7 +169,6 @@ txdb.cpp txdb.h txmempool.cpp txmempool.h -ui_interface.h uint256.cpp uint256.h undo.h diff --git a/src/Makefile.am b/src/Makefile.am index 0be5a3c6..37b2db48 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -92,7 +92,6 @@ BITCOIN_CORE_H = \ networks/netman.h \ networks/network.h \ networks/networktemplate.h \ - noui.h \ policy/fees.h \ policy/policy.h \ pow.h \ @@ -132,9 +131,9 @@ BITCOIN_CORE_H = \ torcontrol.h \ txdb.h \ txmempool.h \ - ui_interface.h \ uint256.h \ undo.h \ + util/logger.h \ util/util.h \ util/utilmoneystr.h \ util/utilstrencodings.h \ @@ -179,7 +178,6 @@ libbitcoin_server_a_SOURCES = \ merkleblock.cpp \ net/net.cpp \ net/netaddress.cpp \ - noui.cpp \ policy/fees.cpp \ policy/policy.cpp \ pow.cpp \ @@ -239,6 +237,7 @@ libbitcoin_server_a_SOURCES = \ rpc/rpcprotocol.cpp \ support/cleanse.cpp \ uint256.cpp \ + util/logger.cpp \ util/util.cpp \ util/utilmoneystr.cpp \ util/utilstrencodings.cpp \ diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index b44373f0..5ec97a59 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -13,7 +13,7 @@ main(int argc, char** argv) { ECC_Start(); SetupEnvironment(); - fPrintToDebugLog = false; // don't want to write to debug.log file + g_logger->fPrintToDebugLog = false; // don't want to write to debug.log file benchmark::BenchRunner::RunAll(); diff --git a/src/blockgeneration/miner.cpp b/src/blockgeneration/miner.cpp index 891a267b..e90017b3 100644 --- a/src/blockgeneration/miner.cpp +++ b/src/blockgeneration/miner.cpp @@ -417,7 +417,7 @@ bool CheckWork(const CBlock *pblock, CWallet &wallet, boost::shared_ptr CreateNewPoSBlock(CWallet *pwallet, const CScrip void EccMinter(CWallet *pwallet) { - LogPrintf("CPUMiner started for proof-of-%s\n", "stake"); + LogPrintf("CPUMiner started for proof-of-stake\n"); SetThreadPriority(THREAD_PRIORITY_LOWEST); // Make this thread recognisable as the mining thread RenameThread("ecc-minter"); diff --git a/src/chain/block.cpp b/src/chain/block.cpp index 0ad23fbe..85710ced 100644 --- a/src/chain/block.cpp +++ b/src/chain/block.cpp @@ -171,17 +171,14 @@ bool CBlock::CheckBlockSignature() const std::vector &vchPubKey = vSolutions[0]; if (vchBlockSig.empty()) { - LogPrintf("sig is empty? \n"); continue; } if (!CPubKey(vchPubKey).Verify(GetHash(), vchBlockSig)) { - LogPrintf("probably will see this message \n"); continue; } return true; } - LogPrintf("got here somehow?"); } } LogPrintf("CheckBlockSignature failed \n"); diff --git a/src/chain/tx.cpp b/src/chain/tx.cpp index 1aa41959..7ee2c33f 100644 --- a/src/chain/tx.cpp +++ b/src/chain/tx.cpp @@ -276,14 +276,18 @@ uint64_t CTransaction::GetCoinAge(uint64_t nCoinAge, bool byValue) const int64_t nValueIn = txPrev.vout[txin.prevout.n].nValue; bnCentSecond += arith_uint256(nValueIn) * (nTime - txPrev.nTime) / CENT; - if (fDebug && gArgs.GetBoolArg("-printcoinage", false)) + if (gArgs.GetBoolArg("-printcoinage", false)) + { LogPrintf("coin age nValueIn=%d nTimeDiff=%d bnCentSecond=%s\n", nValueIn, nTime - txPrev.nTime, bnCentSecond.ToString().c_str()); + } } arith_uint256 bnCoinDay = bnCentSecond * CENT / COIN / (24 * 60 * 60); - if (fDebug && gArgs.GetBoolArg("-printcoinage", false)) + if (gArgs.GetBoolArg("-printcoinage", false)) + { LogPrintf("coin age bnCoinDay=%s\n", bnCoinDay.ToString().c_str()); + } nCoinAge = bnCoinDay.GetLow64(); return nCoinAge; } @@ -325,14 +329,18 @@ bool CTransaction::GetCoinAge(uint64_t &nCoinAge) const int64_t nValueIn = txPrev.vout[txin.prevout.n].nValue; bnCentSecond += arith_uint256(nValueIn) * (nTime - txPrev.nTime) / CENT; - if (fDebug && gArgs.GetBoolArg("-printcoinage", false)) + if (gArgs.GetBoolArg("-printcoinage", false)) + { LogPrintf("coin age nValueIn=%d nTimeDiff=%d bnCentSecond=%s\n", nValueIn, nTime - txPrev.nTime, bnCentSecond.ToString().c_str()); + } } arith_uint256 bnCoinDay = bnCentSecond * CENT / COIN / (24 * 60 * 60); - if (fDebug && gArgs.GetBoolArg("-printcoinage", false)) + if (gArgs.GetBoolArg("-printcoinage", false)) + { LogPrintf("coin age bnCoinDay=%s\n", bnCoinDay.ToString().c_str()); + } nCoinAge = bnCoinDay.GetLow64(); return true; } diff --git a/src/coins.cpp b/src/coins.cpp index c353e83e..7e072f29 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -20,20 +20,14 @@ #include "coins.h" -#include "memusage.h" -#include "random.h" -#include "util/util.h" -#include - -#include "coins.h" - #include "consensus/consensus.h" #include "memusage.h" #include "random.h" +#include "util/logger.h" #include "util/util.h" - #include + bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } diff --git a/src/core_read.cpp b/src/core_read.cpp index c170902a..63a28146 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -24,6 +24,7 @@ #include "script/script.h" #include "serialize.h" #include "streams.h" +#include "util/logger.h" #include "util/util.h" #include "util/utilstrencodings.h" #include "version.h" diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 7740633c..9eb2d348 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -24,6 +24,7 @@ #include "clientversion.h" #include "serialize.h" #include "streams.h" +#include "util/logger.h" #include "util/util.h" #include "util/utilstrencodings.h" #include "version.h" diff --git a/src/eccoind.cpp b/src/eccoind.cpp index 58f824e5..879bef90 100644 --- a/src/eccoind.cpp +++ b/src/eccoind.cpp @@ -23,10 +23,10 @@ #include "httpserver.h" #include "init.h" #include "networks/netman.h" -#include "noui.h" #include "rpc/rpcserver.h" #include "sync.h" #include "threadgroup.h" +#include "util/logger.h" #include "util/util.h" #include @@ -91,7 +91,7 @@ bool AppInit(int argc, char *argv[]) // Process help and version before taking care about datadir if (gArgs.IsArgSet("-?") || gArgs.IsArgSet("-h") || gArgs.IsArgSet("-help") || gArgs.IsArgSet("-version")) { - std::string strUsage = _("Eccoind") + " " + _("version") + " " + FormatFullVersion() + "\n"; + std::string strUsage = "Eccoind version " + FormatFullVersion() + "\n"; if (gArgs.IsArgSet("-version")) { @@ -99,8 +99,7 @@ bool AppInit(int argc, char *argv[]) } else { - strUsage += - "\n" + _("Usage:") + "\n" + " eccoind [options] " + _("Start Eccoind") + "\n"; + strUsage += "\nUsage:\neccoind [options] Start Eccoind\n"; strUsage += "\n" + HelpMessage(); } @@ -187,8 +186,5 @@ int main(int argc, char *argv[]) { SetupEnvironment(); - // Connect eccoind signal handlers - noui_connect(); - return (AppInit(argc, argv) ? 0 : 1); } diff --git a/src/httprpc.cpp b/src/httprpc.cpp index decf0004..b76b79e1 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -29,7 +29,7 @@ #include "rpc/rpcprotocol.h" #include "rpc/rpcserver.h" #include "sync.h" -#include "ui_interface.h" + #include "util/util.h" #include "util/utilstrencodings.h" #include "util/utilstrencodings.h" @@ -245,9 +245,7 @@ static bool InitRPCAuthentication() LogPrintf("No rpcpassword set - using random cookie authentication\n"); if (!GenerateAuthCookie(&strRPCUserColonPass)) { - uiInterface.ThreadSafeMessageBox( - _("Error: A fatal internal error occurred, see debug.log for details"), // Same message as AbortNode - "", CClientUIInterface::MSG_ERROR); + LogPrintf("Error: A fatal internal error occurred, see debug.log for details\n"); return false; } } diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 4a0c22b2..cafdd270 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -27,7 +27,7 @@ #include "networks/netman.h" #include "rpc/rpcprotocol.h" // For HTTP status codes #include "sync.h" -#include "ui_interface.h" + #include "util/util.h" #include @@ -234,12 +234,11 @@ static bool InitHTTPAllowList() LookupSubNet(strAllow.c_str(), subnet); if (!subnet.IsValid()) { - uiInterface.ThreadSafeMessageBox(strprintf("Invalid -rpcallowip subnet specification: %s. " - "Valid are a single IP (e.g. 1.2.3.4), a " - "network/netmask (e.g. 1.2.3.4/255.255.255.0) or " - "a network/CIDR (e.g. 1.2.3.4/24).", - strAllow), - "", CClientUIInterface::MSG_ERROR); + LogPrintf("Invalid -rpcallowip subnet specification: %s. " + "Valid are a single IP (e.g. 1.2.3.4), a " + "network/netmask (e.g. 1.2.3.4/255.255.255.0) or " + "a network/CIDR (e.g. 1.2.3.4/24).", + strAllow); return false; } rpc_allow_subnets.push_back(subnet); @@ -430,8 +429,7 @@ bool InitHTTPServer() if (gArgs.GetBoolArg("-rpcssl", false)) { - uiInterface.ThreadSafeMessageBox( - "SSL mode for RPC (-rpcssl) is no longer supported.", "", CClientUIInterface::MSG_ERROR); + LogPrintf("SSL mode for RPC (-rpcssl) is no longer supported. \n"); return false; } @@ -440,7 +438,7 @@ bool InitHTTPServer() #if LIBEVENT_VERSION_NUMBER >= 0x02010100 // If -debug=libevent, set full libevent debugging. // Otherwise, disable all libevent debugging. - if (LogAcceptCategory("libevent")) + if (g_logger->LogAcceptCategory("libevent")) event_enable_debug_logging(EVENT_DBG_ALL); else event_enable_debug_logging(EVENT_DBG_NONE); diff --git a/src/init.cpp b/src/init.cpp index d131f7bb..159ff0f9 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -44,7 +44,6 @@ #include "torcontrol.h" #include "txdb.h" #include "txmempool.h" -#include "ui_interface.h" #include "util/util.h" #include "util/utilmoneystr.h" #include "util/utilstrencodings.h" @@ -104,7 +103,6 @@ enum BindFlags }; static const char *FEE_ESTIMATES_FILENAME = "fee_estimates.dat"; -CClientUIInterface uiInterface; // Declared but not defined in ui_interface.h ////////////////////////////////////////////////////////////////////////////// // @@ -159,8 +157,6 @@ class CCoinsViewErrorCatcher final : public CCoinsViewBacked } catch (const std::runtime_error &e) { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); // Starting the shutdown sequence and returning false to the caller would be // interpreted as 'entry not found' (as opposed to unable to read data), and @@ -283,6 +279,8 @@ void Shutdown() globalVerifyHandle.reset(); ECC_Stop(); LogPrintf("%s: done\n", __func__); + delete g_logger; + g_logger = nullptr; } /** @@ -292,13 +290,13 @@ void HandleSIGTERM(int) { shutdown_threads.store(true); } void HandleSIGHUP(int) { fReopenDebugLog = true; } bool static InitError(const std::string &str) { - uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_ERROR); + LogPrintf("InitError: %s\n", str.c_str()); return false; } bool static InitWarning(const std::string &str) { - uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_WARNING); + LogPrintf("InitWarning: %s\n", str.c_str()); return true; } @@ -337,166 +335,161 @@ std::string HelpMessage() // When adding new options to the categories, please keep and ensure alphabetical ordering. // Do not translate _(...) -help-debug options, Many technical terms, and only a very small audience, so is // unnecessary stress to translators. - std::string strUsage = HelpMessageGroup(_("Options:")); - strUsage += HelpMessageOpt("-?", _("This help message")); - strUsage += HelpMessageOpt("-version", _("Print version and exit")); + std::string strUsage = HelpMessageGroup(("Options:")); + strUsage += HelpMessageOpt("-?", ("This help message")); + strUsage += HelpMessageOpt("-version", ("Print version and exit")); strUsage += HelpMessageOpt( - "-blocknotify=", _("Execute command when the best block changes (%s in cmd is replaced by block hash)")); + "-blocknotify=", ("Execute command when the best block changes (%s in cmd is replaced by block hash)")); strUsage += HelpMessageOpt("-checkblocks=", - strprintf(_("How many blocks to check at startup (default: %u, 0 = all)"), DEFAULT_CHECKBLOCKS)); + strprintf(("How many blocks to check at startup (default: %u, 0 = all)"), DEFAULT_CHECKBLOCKS)); strUsage += HelpMessageOpt("-checklevel=", - strprintf(_("How thorough the block verification of -checkblocks is (0-4, default: %u)"), DEFAULT_CHECKLEVEL)); - strUsage += HelpMessageOpt("-conf=", strprintf(_("Specify configuration file (default: %s)"), CONF_FILENAME)); + strprintf(("How thorough the block verification of -checkblocks is (0-4, default: %u)"), DEFAULT_CHECKLEVEL)); + strUsage += HelpMessageOpt("-conf=", strprintf(("Specify configuration file (default: %s)"), CONF_FILENAME)); strUsage += HelpMessageOpt("-returnchange", - strprintf(_("Specify if change is returned to same address (default: %u)"), DEFAULT_RETURN_CHANGE)); + strprintf(("Specify if change is returned to same address (default: %u)"), DEFAULT_RETURN_CHANGE)); { #ifndef WIN32 - strUsage += HelpMessageOpt("-daemon", _("Run in the background as a daemon and accept commands")); + strUsage += HelpMessageOpt("-daemon", ("Run in the background as a daemon and accept commands")); #endif } - strUsage += HelpMessageOpt("-datadir=", _("Specify data directory")); + strUsage += HelpMessageOpt("-datadir=", ("Specify data directory")); strUsage += - HelpMessageOpt("-dbcache=", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), + HelpMessageOpt("-dbcache=", strprintf(("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache)); - strUsage += HelpMessageOpt("-loadblock=", _("Imports blocks from external blk000??.dat file on startup")); + strUsage += HelpMessageOpt("-loadblock=", ("Imports blocks from external blk000??.dat file on startup")); strUsage += HelpMessageOpt( - "-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), + "-maxorphantx=", strprintf(("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=", - strprintf(_("Keep the transaction memory pool below megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); - strUsage += HelpMessageOpt("-mempoolexpiry=", - strprintf(_("Do not keep transactions in the mempool longer than hours (default: %u)"), - DEFAULT_MEMPOOL_EXPIRY)); - strUsage += HelpMessageOpt("-par=", strprintf(_("Set the number of script verification threads (%u to %d, 0 = " - "auto, <0 = leave that many cores free, default: %d)"), + strprintf(("Keep the transaction memory pool below megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); + strUsage += HelpMessageOpt( + "-mempoolexpiry=", strprintf(("Do not keep transactions in the mempool longer than hours (default: %u)"), + DEFAULT_MEMPOOL_EXPIRY)); + strUsage += HelpMessageOpt("-par=", strprintf(("Set the number of script verification threads (%u to %d, 0 = " + "auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); #ifndef WIN32 - strUsage += HelpMessageOpt("-pid=", strprintf(_("Specify pid file (default: %s)"), PID_FILENAME)); + strUsage += HelpMessageOpt("-pid=", strprintf(("Specify pid file (default: %s)"), PID_FILENAME)); #endif - strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files on startup")); + strUsage += HelpMessageOpt("-reindex", ("Rebuild block chain index from current blk000??.dat files on startup")); - strUsage += HelpMessageGroup(_("Connection options:")); - strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); + strUsage += HelpMessageGroup(("Connection options:")); + strUsage += HelpMessageOpt("-addnode=", ("Add a node to connect to and attempt to keep the connection open")); strUsage += HelpMessageOpt("-banscore=", - strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); + strprintf(("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt( - "-bantime=", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), + "-bantime=", strprintf(("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt( - "-bind=", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=", _("Connect only to the specified node(s)")); + "-bind=", ("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); + strUsage += HelpMessageOpt("-connect=", ("Connect only to the specified node(s)")); strUsage += HelpMessageOpt( - "-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); - strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + - strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); + "-discover", ("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); + strUsage += HelpMessageOpt("-dns", + "Allow DNS lookups for -addnode, -seednode and -connect " + strprintf("(default: %u)", DEFAULT_NAME_LOOKUP)); strUsage += HelpMessageOpt( - "-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect)")); - strUsage += HelpMessageOpt("-externalip=", _("Specify your own public address")); + "-dnsseed", ("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect)")); + strUsage += HelpMessageOpt("-externalip=", ("Specify your own public address")); strUsage += HelpMessageOpt("-forcednsseed", - strprintf(_("Always query for peer addresses via DNS lookup (default: %u)"), DEFAULT_FORCEDNSSEED)); - strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect)")); + strprintf(("Always query for peer addresses via DNS lookup (default: %u)"), DEFAULT_FORCEDNSSEED)); + strUsage += HelpMessageOpt("-listen", ("Accept connections from outside (default: 1 if no -proxy or -connect)")); strUsage += HelpMessageOpt( - "-listenonion", strprintf(_("Automatically create Tor hidden service (default: %d)"), DEFAULT_LISTEN_ONION)); + "-listenonion", strprintf(("Automatically create Tor hidden service (default: %d)"), DEFAULT_LISTEN_ONION)); strUsage += HelpMessageOpt("-maxconnections=", - strprintf(_("Maintain at most connections to peers (default: %u)"), DEFAULT_MAX_PEER_CONNECTIONS)); + strprintf(("Maintain at most connections to peers (default: %u)"), DEFAULT_MAX_PEER_CONNECTIONS)); strUsage += HelpMessageOpt("-maxreceivebuffer=", - strprintf(_("Maximum per-connection receive buffer, *1000 bytes (default: %u)"), DEFAULT_MAXRECEIVEBUFFER)); + strprintf(("Maximum per-connection receive buffer, *1000 bytes (default: %u)"), DEFAULT_MAXRECEIVEBUFFER)); strUsage += HelpMessageOpt("-maxsendbuffer=", - strprintf(_("Maximum per-connection send buffer, *1000 bytes (default: %u)"), DEFAULT_MAXSENDBUFFER)); + strprintf(("Maximum per-connection send buffer, *1000 bytes (default: %u)"), DEFAULT_MAXSENDBUFFER)); strUsage += HelpMessageOpt("-onion=", - strprintf(_("Use separate SOCKS5 proxy to reach peers via Tor hidden services (default: %s)"), "-proxy")); - strUsage += HelpMessageOpt("-onlynet=", _("Only connect to nodes in network (ipv4, ipv6 or onion)")); + strprintf(("Use separate SOCKS5 proxy to reach peers via Tor hidden services (default: %s)"), "-proxy")); + strUsage += HelpMessageOpt("-onlynet=", ("Only connect to nodes in network (ipv4, ipv6 or onion)")); strUsage += HelpMessageOpt( - "-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %u)"), DEFAULT_PERMIT_BAREMULTISIG)); + "-permitbaremultisig", strprintf(("Relay non-P2SH multisig (default: %u)"), DEFAULT_PERMIT_BAREMULTISIG)); strUsage += HelpMessageOpt("-peerbloomfilters", - strprintf(_("Support filtering of blocks and transaction with bloom filters (default: %u)"), 1)); - strUsage += HelpMessageOpt("-port=", strprintf(_("Listen for connections on (default: %u)"), + strprintf(("Support filtering of blocks and transaction with bloom filters (default: %u)"), 1)); + strUsage += HelpMessageOpt("-port=", strprintf(("Listen for connections on (default: %u)"), pnetMan->getActivePaymentNetwork()->GetDefaultPort())); - strUsage += HelpMessageOpt("-proxy=", _("Connect through SOCKS5 proxy")); - strUsage += HelpMessageOpt( - "-proxyrandomize", - strprintf( - _("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), - DEFAULT_PROXYRANDOMIZE)); - strUsage += HelpMessageOpt("-seednode=", _("Connect to a node to retrieve peer addresses, and disconnect")); + strUsage += HelpMessageOpt("-proxy=", ("Connect through SOCKS5 proxy")); + strUsage += HelpMessageOpt("-proxyrandomize", + strprintf(("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), + DEFAULT_PROXYRANDOMIZE)); + strUsage += HelpMessageOpt("-seednode=", ("Connect to a node to retrieve peer addresses, and disconnect")); strUsage += HelpMessageOpt("-timeout=", - strprintf(_("Specify connection timeout in milliseconds (minimum: 1, default: %d)"), DEFAULT_CONNECT_TIMEOUT)); + strprintf(("Specify connection timeout in milliseconds (minimum: 1, default: %d)"), DEFAULT_CONNECT_TIMEOUT)); strUsage += HelpMessageOpt("-torcontrol=:", - strprintf(_("Tor control port to use if onion listening enabled (default: %s)"), DEFAULT_TOR_CONTROL)); - strUsage += HelpMessageOpt("-torpassword=", _("Tor control port password (default: empty)")); + strprintf(("Tor control port to use if onion listening enabled (default: %s)"), DEFAULT_TOR_CONTROL)); + strUsage += HelpMessageOpt("-torpassword=", ("Tor control port password (default: empty)")); #ifdef USE_UPNP #if USE_UPNP strUsage += - HelpMessageOpt("-upnp", _("Use UPnP to map the listening port (default: 1 when listening and no -proxy)")); + HelpMessageOpt("-upnp", ("Use UPnP to map the listening port (default: 1 when listening and no -proxy)")); #else - strUsage += HelpMessageOpt("-upnp", strprintf(_("Use UPnP to map the listening port (default: %u)"), 0)); + strUsage += HelpMessageOpt("-upnp", strprintf(("Use UPnP to map the listening port (default: %u)"), 0)); #endif #endif strUsage += HelpMessageOpt("-whitebind=", - _("Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6")); + ("Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6")); strUsage += HelpMessageOpt("-whitelist=", - _("Whitelist peers connecting from the given netmask or IP address. Can be specified multiple times.") + " " + - _("Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are " - "already in the mempool, useful e.g. for a gateway")); + "Whitelist peers connecting from the given netmask or IP address. Can be specified multiple times. " + "Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are " + "already in the mempool, useful e.g. for a gateway"); strUsage += - HelpMessageOpt("-whitelistrelay", strprintf(_("Accept relayed transactions received from whitelisted peers " - "even when not relaying transactions (default: %d)"), + HelpMessageOpt("-whitelistrelay", strprintf(("Accept relayed transactions received from whitelisted peers " + "even when not relaying transactions (default: %d)"), DEFAULT_WHITELISTRELAY)); strUsage += HelpMessageOpt( "-whitelistforcerelay", strprintf( - _("Force relay of transactions from whitelisted peers even they violate local relay policy (default: %d)"), + ("Force relay of transactions from whitelisted peers even they violate local relay policy (default: %d)"), DEFAULT_WHITELISTFORCERELAY)); strUsage += HelpMessageOpt( "-maxuploadtarget=", strprintf( - _("Tries to keep outbound traffic under the given target (in MiB per 24h), 0 = no limit (default: %d)"), + ("Tries to keep outbound traffic under the given target (in MiB per 24h), 0 = no limit (default: %d)"), DEFAULT_MAX_UPLOAD_TARGET)); - strUsage += HelpMessageGroup(_("Wallet options:")); + strUsage += HelpMessageGroup(("Wallet options:")); strUsage += - HelpMessageOpt("-keypool=", strprintf(_("Set key pool size to (default: %u)"), DEFAULT_KEYPOOL_SIZE)); + HelpMessageOpt("-keypool=", strprintf(("Set key pool size to (default: %u)"), DEFAULT_KEYPOOL_SIZE)); strUsage += HelpMessageOpt("-fallbackfee=", - strprintf(_("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data (default: %s)"), + strprintf(("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE))); strUsage += HelpMessageOpt("-mintxfee=", - strprintf(_("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)"), + strprintf(("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MINFEE))); strUsage += - HelpMessageOpt("-paytxfee=", strprintf(_("Fee (in %s/kB) to add to transactions you send (default: %s)"), + HelpMessageOpt("-paytxfee=", strprintf(("Fee (in %s/kB) to add to transactions you send (default: %s)"), CURRENCY_UNIT, FormatMoney(payTxFee.GetFeePerK()))); - strUsage += HelpMessageOpt("-rescan", _("Rescan the block chain for missing wallet transactions on startup")); + strUsage += HelpMessageOpt("-rescan", ("Rescan the block chain for missing wallet transactions on startup")); strUsage += - HelpMessageOpt("-salvagewallet", _("Attempt to recover private keys from a corrupt wallet.dat on startup")); + HelpMessageOpt("-salvagewallet", ("Attempt to recover private keys from a corrupt wallet.dat on startup")); strUsage += HelpMessageOpt( - "-sendfreetransactions", strprintf(_("Send transactions as zero-fee transactions if possible (default: %u)"), + "-sendfreetransactions", strprintf(("Send transactions as zero-fee transactions if possible (default: %u)"), DEFAULT_SEND_FREE_TRANSACTIONS)); + strUsage += HelpMessageOpt("-spendzeroconfchange", + strprintf(("Spend unconfirmed change when sending transactions (default: %u)"), DEFAULT_SPEND_ZEROCONF_CHANGE)); strUsage += HelpMessageOpt( - "-spendzeroconfchange", strprintf(_("Spend unconfirmed change when sending transactions (default: %u)"), - DEFAULT_SPEND_ZEROCONF_CHANGE)); - strUsage += HelpMessageOpt( - "-txconfirmtarget=", strprintf(_("If paytxfee is not set, include enough fee so transactions begin " - "confirmation on average within n blocks (default: %u)"), + "-txconfirmtarget=", strprintf(("If paytxfee is not set, include enough fee so transactions begin " + "confirmation on average within n blocks (default: %u)"), DEFAULT_TX_CONFIRM_TARGET)); strUsage += HelpMessageOpt( - "-maxtxfee=", strprintf(_("Maximum total fees (in %s) to use in a single wallet transaction; setting this " - "too low may abort large transactions (default: %s)"), + "-maxtxfee=", strprintf(("Maximum total fees (in %s) to use in a single wallet transaction; setting this " + "too low may abort large transactions (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE))); - strUsage += HelpMessageOpt("-upgradewallet", _("Upgrade wallet to latest format on startup")); - strUsage += HelpMessageOpt("-wallet=", - _("Specify wallet file (within data directory)") + " " + strprintf(_("(default: %s)"), "wallet.dat")); + strUsage += HelpMessageOpt("-upgradewallet", ("Upgrade wallet to latest format on startup")); + strUsage += HelpMessageOpt("-wallet=", "Specify wallet file (within data directory) (default: wallet.dat)"); strUsage += HelpMessageOpt("-walletbroadcast", - _("Make the wallet broadcast transactions") + " " + strprintf(_("(default: %u)"), DEFAULT_WALLETBROADCAST)); + "Make the wallet broadcast transactions " + strprintf("(default: %u)", DEFAULT_WALLETBROADCAST)); strUsage += HelpMessageOpt( - "-walletnotify=", _("Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)")); + "-walletnotify=", ("Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)")); strUsage += HelpMessageOpt("-zapwallettxes=", - _("Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup") + - " " + - _("(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)")); + "Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup " + "(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)"); - strUsage += HelpMessageGroup(_("Debugging/Testing options:")); - strUsage += HelpMessageOpt("-uacomment=", _("Append comment to the user agent string")); + strUsage += HelpMessageGroup(("Debugging/Testing options:")); + strUsage += HelpMessageOpt("-uacomment=", ("Append comment to the user agent string")); if (showDebug) { strUsage += HelpMessageOpt("-checkblockindex", @@ -545,17 +538,17 @@ std::string HelpMessage() "mempoolrej, net, proxy, http, libevent, tor, zmq"; debugCategories += ", qt"; strUsage += HelpMessageOpt("-debug=", - strprintf(_("Output debugging information (default: %u, supplying is optional)"), 0) + ". " + - _("If is not supplied or if = 1, output all debugging information.") + - _(" can be:") + " " + debugCategories + "."); + strprintf(("Output debugging information (default: %u, supplying is optional)"), 0) + ". " + + ("If is not supplied or if = 1, output all debugging information.") + + (" can be:") + " " + debugCategories + "."); if (showDebug) strUsage += HelpMessageOpt("-nodebug", "Turn off debugging messages, same as -debug=0"); - strUsage += HelpMessageOpt("-staking", strprintf(_("Generate coins (default: %u)"), DEFAULT_GENERATE)); - strUsage += HelpMessageOpt("-help-debug", _("Show all debugging options (usage: --help -help-debug)")); + strUsage += HelpMessageOpt("-staking", strprintf(("Generate coins (default: %u)"), DEFAULT_GENERATE)); + strUsage += HelpMessageOpt("-help-debug", ("Show all debugging options (usage: --help -help-debug)")); strUsage += - HelpMessageOpt("-logips", strprintf(_("Include IP addresses in debug output (default: %u)"), DEFAULT_LOGIPS)); + HelpMessageOpt("-logips", strprintf(("Include IP addresses in debug output (default: %u)"), DEFAULT_LOGIPS)); strUsage += HelpMessageOpt( - "-logtimestamps", strprintf(_("Prepend debug output with timestamp (default: %u)"), DEFAULT_LOGTIMESTAMPS)); + "-logtimestamps", strprintf(("Prepend debug output with timestamp (default: %u)"), DEFAULT_LOGTIMESTAMPS)); if (showDebug) { strUsage += HelpMessageOpt("-logtimemicros", @@ -568,10 +561,10 @@ std::string HelpMessage() strprintf("Limit size of signature cache to MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE)); } strUsage += HelpMessageOpt( - "-minrelaytxfee=", strprintf(_("Fees (in %s/kB) smaller than this are considered zero fee for relaying, " - "mining and transaction creation (default: %s)"), + "-minrelaytxfee=", strprintf(("Fees (in %s/kB) smaller than this are considered zero fee for relaying, " + "mining and transaction creation (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE))); - strUsage += HelpMessageOpt("-printtoconsole", _("Send trace/debug info to console instead of debug.log file")); + strUsage += HelpMessageOpt("-printtoconsole", ("Send trace/debug info to console instead of debug.log file")); if (showDebug) { strUsage += HelpMessageOpt( @@ -582,7 +575,7 @@ std::string HelpMessage() strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB)); } - strUsage += HelpMessageGroup(_("Node relay options:")); + strUsage += HelpMessageGroup(("Node relay options:")); /// TODO: fix this as it is temporarily disabled /* @@ -591,40 +584,39 @@ std::string HelpMessage() (%sdefault: %u)", "testnet/regtest only; ", !Params(CNetMan::TESTNET).RequireStandard())); */ strUsage += HelpMessageOpt( - "-bytespersigop", strprintf(_("Minimum bytes per sigop in transactions we relay and mine (default: %u)"), + "-bytespersigop", strprintf(("Minimum bytes per sigop in transactions we relay and mine (default: %u)"), DEFAULT_BYTES_PER_SIGOP)); strUsage += HelpMessageOpt("-datacarrier", - strprintf(_("Relay and mine data carrier transactions (default: %u)"), DEFAULT_ACCEPT_DATACARRIER)); + strprintf(("Relay and mine data carrier transactions (default: %u)"), DEFAULT_ACCEPT_DATACARRIER)); strUsage += HelpMessageOpt("-datacarriersize", - strprintf(_("Maximum size of data in data carrier transactions we relay and mine (default: %u)"), + strprintf(("Maximum size of data in data carrier transactions we relay and mine (default: %u)"), MAX_OP_RETURN_RELAY)); - strUsage += HelpMessageGroup(_("Block creation options:")); + strUsage += HelpMessageGroup(("Block creation options:")); if (showDebug) strUsage += HelpMessageOpt("-blockversion=", "Override block version to test forking scenarios"); - strUsage += HelpMessageGroup(_("RPC server options:")); - strUsage += HelpMessageOpt("-server", _("Accept command line and JSON-RPC commands")); - strUsage += HelpMessageOpt("-rest", strprintf(_("Accept public REST requests (default: %u)"), DEFAULT_REST_ENABLE)); + strUsage += HelpMessageGroup(("RPC server options:")); + strUsage += HelpMessageOpt("-server", ("Accept command line and JSON-RPC commands")); + strUsage += HelpMessageOpt("-rest", strprintf(("Accept public REST requests (default: %u)"), DEFAULT_REST_ENABLE)); strUsage += HelpMessageOpt( - "-rpcbind=", _("Bind to given address to listen for JSON-RPC connections. Use [host]:port notation for " - "IPv6. This option can be specified multiple times (default: bind to all interfaces)")); - strUsage += HelpMessageOpt("-rpccookiefile=", _("Location of the auth cookie (default: data dir)")); - strUsage += HelpMessageOpt("-rpcuser=", _("Username for JSON-RPC connections")); - strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); + "-rpcbind=", ("Bind to given address to listen for JSON-RPC connections. Use [host]:port notation for " + "IPv6. This option can be specified multiple times (default: bind to all interfaces)")); + strUsage += HelpMessageOpt("-rpccookiefile=", ("Location of the auth cookie (default: data dir)")); + strUsage += HelpMessageOpt("-rpcuser=", ("Username for JSON-RPC connections")); + strUsage += HelpMessageOpt("-rpcpassword=", ("Password for JSON-RPC connections")); strUsage += HelpMessageOpt( - "-rpcauth=", _("Username and hashed password for JSON-RPC connections. The field comes in the " - "format: :$. A canonical python script is included in " - "share/rpcuser. This option can be specified multiple times")); - strUsage += - HelpMessageOpt("-rpcport=", strprintf(_("Listen for JSON-RPC connections on (default: %u)"), - pnetMan->getActivePaymentNetwork()->GetRPCPort())); + "-rpcauth=", ("Username and hashed password for JSON-RPC connections. The field comes in the " + "format: :$. A canonical python script is included in " + "share/rpcuser. This option can be specified multiple times")); + strUsage += HelpMessageOpt("-rpcport=", strprintf(("Listen for JSON-RPC connections on (default: %u)"), + pnetMan->getActivePaymentNetwork()->GetRPCPort())); strUsage += HelpMessageOpt( - "-rpcallowip=", _("Allow JSON-RPC connections from specified source. Valid for are a single IP (e.g. " - "1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. " - "1.2.3.4/24). This option can be specified multiple times")); + "-rpcallowip=", ("Allow JSON-RPC connections from specified source. Valid for are a single IP (e.g. " + "1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. " + "1.2.3.4/24). This option can be specified multiple times")); strUsage += HelpMessageOpt("-rpcthreads=", - strprintf(_("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); + strprintf(("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); if (showDebug) { strUsage += HelpMessageOpt("-rpcworkqueue=", @@ -639,18 +631,23 @@ std::string HelpMessage() std::string LicenseInfo() { // todo: remove urls from translations on next change - return FormatParagraph(strprintf(_("Copyright (C) 2014-%i The Eccoin Developers"), COPYRIGHT_YEAR)) + "\n" + "\n" + - FormatParagraph(_("This is experimental software.")) + "\n" + "\n" + - FormatParagraph(_("Distributed under the MIT software license, see the accompanying file COPYING or " - ".")) + - "\n" + "\n" + FormatParagraph(_("This product includes software developed by the OpenSSL Project for use in " - "the OpenSSL Toolkit and cryptographic software " - "written by Eric Young and UPnP software written by Thomas Bernard.")) + + return FormatParagraph(strprintf(("Copyright (C) 2014-%i The Eccoin Developers"), COPYRIGHT_YEAR)) + "\n" + "\n" + + FormatParagraph(("This is experimental software.")) + "\n" + "\n" + + FormatParagraph(("Distributed under the MIT software license, see the accompanying file COPYING or " + ".")) + + "\n" + "\n" + FormatParagraph(("This product includes software developed by the OpenSSL Project for use in " + "the OpenSSL Toolkit and cryptographic software " + "written by Eric Young and UPnP software written by Thomas Bernard.")) + "\n"; } -static void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex) +void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex) { + if (!gArgs.IsArgSet("-blocknotify")) + { + return; + } + if (initialSync || !pBlockIndex) return; @@ -860,15 +857,20 @@ void InitParameterInteraction() static std::string ResolveErrMsg(const char *const optname, const std::string &strBind) { - return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); + return strprintf(("Cannot resolve -%s address: '%s'"), optname, strBind); } void InitLogging() { - fPrintToConsole = gArgs.GetBoolArg("-printtoconsole", false); - fLogTimestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); - fLogTimeMicros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); - fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); + g_logger->fPrintToConsole = gArgs.GetBoolArg("-printtoconsole", false); + g_logger->fLogTimestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); + g_logger->fLogTimeMicros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); + g_logger->fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); + + if (g_logger->fPrintToDebugLog) + { + g_logger->OpenDebugLog(); + } LogPrintf("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"); LogPrintf("Eccoin version %s (%s)\n", FormatFullVersion(), CLIENT_DATE); @@ -957,7 +959,7 @@ bool AppInit2(thread_group &threadGroup) initFD = RaiseFileDescriptorLimit(initMaxConnections + MIN_CORE_FILEDESCRIPTORS); if (initFD < MIN_CORE_FILEDESCRIPTORS) { - return InitError(_("Not enough file descriptors available.")); + return InitError(("Not enough file descriptors available.")); } initMaxConnections = std::min(initFD - MIN_CORE_FILEDESCRIPTORS, initMaxConnections); @@ -965,36 +967,36 @@ bool AppInit2(thread_group &threadGroup) { LogPrintf("Reducing -maxconnections from %d to %d, because of system limitations.", nUserMaxConnections, initMaxConnections); - InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), + InitWarning(strprintf(("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, initMaxConnections)); } // ********************************************************* Step 3: parameter-to-internal-flags - fDebug = gArgs.IsArgSet("-debug"); + g_logger->fDebug = gArgs.IsArgSet("-debug"); // Special-case: if -debug=0/-nodebug is set, turn off debugging messages const std::vector &categories = gArgs.GetArgs("-debug"); if (gArgs.GetBoolArg("-nodebug", false) || find(categories.begin(), categories.end(), std::string("0")) != categories.end()) - fDebug = false; + g_logger->fDebug = false; // Check for -debugnet if (gArgs.GetBoolArg("-debugnet", false)) - InitWarning(_("Unsupported argument -debugnet ignored, use -debug=net.")); + InitWarning(("Unsupported argument -debugnet ignored, use -debug=net.")); // Check for -socks - as this is a privacy risk to continue, exit here if (gArgs.IsArgSet("-socks")) - return InitError(_("Unsupported argument -socks found. Setting SOCKS version isn't possible anymore, only " - "SOCKS5 proxies are supported.")); + return InitError(("Unsupported argument -socks found. Setting SOCKS version isn't possible anymore, only " + "SOCKS5 proxies are supported.")); // Check for -tor - as this is a privacy risk to continue, exit here if (gArgs.GetBoolArg("-tor", false)) - return InitError(_("Unsupported argument -tor found, use -onion.")); + return InitError(("Unsupported argument -tor found, use -onion.")); if (gArgs.GetBoolArg("-benchmark", false)) - InitWarning(_("Unsupported argument -benchmark ignored, use -debug=bench.")); + InitWarning(("Unsupported argument -benchmark ignored, use -debug=bench.")); if (gArgs.GetBoolArg("-whitelistalwaysrelay", false)) InitWarning( - _("Unsupported argument -whitelistalwaysrelay ignored, use -whitelistrelay and/or -whitelistforcerelay.")); + ("Unsupported argument -whitelistalwaysrelay ignored, use -whitelistrelay and/or -whitelistforcerelay.")); // Checkmempool and checkblockindex default to true in regtest mode int ratio = std::min( @@ -1010,7 +1012,7 @@ bool AppInit2(thread_group &threadGroup) int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t nMempoolSizeMin = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; if (nMempoolSizeMax < 0 || nMempoolSizeMax < nMempoolSizeMin) - return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); + return InitError(strprintf(("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); // -par=0 means autodetect, but nScriptCheckThreads==0 means no concurrency nScriptCheckThreads = gArgs.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); @@ -1040,7 +1042,7 @@ bool AppInit2(thread_group &threadGroup) if (ParseMoney(minrelay, n) && n > 0) ::minRelayTxFee = CFeeRate(n); else - return InitError(strprintf(_("Invalid amount for -minrelaytxfee=: '%i'"), + return InitError(strprintf(("Invalid amount for -minrelaytxfee=: '%i'"), gArgs.GetArg("-minrelaytxfee", DEFAULT_TRANSACTION_MINFEE))); } @@ -1058,7 +1060,7 @@ bool AppInit2(thread_group &threadGroup) if (ParseMoney(minfee, n) && n > 0) CWallet::minTxFee = CFeeRate(n); else - return InitError(strprintf(_("Invalid amount for -mintxfee=: '%i'"), + return InitError(strprintf(("Invalid amount for -mintxfee=: '%i'"), gArgs.GetArg("-mintxfee", DEFAULT_TRANSACTION_MINFEE))); } if (gArgs.IsArgSet("-fallbackfee")) @@ -1066,11 +1068,11 @@ bool AppInit2(thread_group &threadGroup) CAmount nFeePerK = 0; std::string fallback = gArgs.GetArg("-fallbackfee", std::to_string(DEFAULT_TRANSACTION_MINFEE)); if (!ParseMoney(fallback, nFeePerK)) - return InitError(strprintf(_("Invalid amount for -fallbackfee=: '%s'"), + return InitError(strprintf(("Invalid amount for -fallbackfee=: '%s'"), gArgs.GetArg("-fallbackfee", DEFAULT_TRANSACTION_MINFEE))); if (nFeePerK > nHighTransactionFeeWarning) - InitWarning(_("-fallbackfee is set very high! This is the transaction fee you may pay when fee estimates " - "are not available.")); + InitWarning(("-fallbackfee is set very high! This is the transaction fee you may pay when fee estimates " + "are not available.")); CWallet::fallbackFee = CFeeRate(nFeePerK); } if (gArgs.IsArgSet("-paytxfee")) @@ -1078,15 +1080,15 @@ bool AppInit2(thread_group &threadGroup) CAmount nFeePerK = 0; std::string payfee = gArgs.GetArg("-paytxfee", std::to_string(DEFAULT_TRANSACTION_MINFEE)); if (!ParseMoney(payfee, nFeePerK)) - return InitError(strprintf(_("Invalid amount for -paytxfee=: '%i'"), + return InitError(strprintf(("Invalid amount for -paytxfee=: '%i'"), gArgs.GetArg("-paytxfee", DEFAULT_TRANSACTION_MINFEE))); if (nFeePerK > nHighTransactionFeeWarning) InitWarning( - _("-paytxfee is set very high! This is the transaction fee you will pay if you send a transaction.")); + ("-paytxfee is set very high! This is the transaction fee you will pay if you send a transaction.")); payTxFee = CFeeRate(nFeePerK, 1000); if (payTxFee < ::minRelayTxFee) { - return InitError(strprintf(_("Invalid amount for -paytxfee=: '%i' (must be at least %s)"), + return InitError(strprintf(("Invalid amount for -paytxfee=: '%i' (must be at least %s)"), gArgs.GetArg("-paytxfee", DEFAULT_TRANSACTION_MINFEE), ::minRelayTxFee.ToString())); } } @@ -1095,15 +1097,15 @@ bool AppInit2(thread_group &threadGroup) CAmount nMaxFee = 0; std::string maxfee = gArgs.GetArg("-maxtxfee", std::to_string(DEFAULT_TRANSACTION_MINFEE)); if (!ParseMoney(maxfee, nMaxFee)) - return InitError(strprintf(_("Invalid amount for -maxtxfee=: '%i'"), + return InitError(strprintf(("Invalid amount for -maxtxfee=: '%i'"), gArgs.GetArg("-maxtxfee", DEFAULT_TRANSACTION_MAXFEE))); if (nMaxFee > HIGH_MAX_TX_FEE) - InitWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); + InitWarning(("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); maxTxFee = nMaxFee; if (CFeeRate(maxTxFee, 1000) < ::minRelayTxFee) { - return InitError(strprintf(_("Invalid amount for -maxtxfee=: '%i' (must be at least the minrelay " - "fee of %s to prevent stuck transactions)"), + return InitError(strprintf(("Invalid amount for -maxtxfee=: '%i' (must be at least the minrelay " + "fee of %s to prevent stuck transactions)"), gArgs.GetArg("-maxtxfee", DEFAULT_TRANSACTION_MAXFEE), ::minRelayTxFee.ToString())); } } @@ -1129,13 +1131,13 @@ bool AppInit2(thread_group &threadGroup) // Sanity check if (!InitSanityCheck()) - return InitError(_("Initialization sanity check failed. Bitcoin Core is shutting down.")); + return InitError(("Initialization sanity check failed. Bitcoin Core is shutting down.")); std::string strDataDir = GetDataDir().string(); // Wallet file must be a plain filename without a directory if (strWalletFile != fs::basename(strWalletFile) + fs::extension(strWalletFile)) - return InitError(strprintf(_("Wallet %s resides outside data directory %s"), strWalletFile, strDataDir)); + return InitError(strprintf(("Wallet %s resides outside data directory %s"), strWalletFile, strDataDir)); // Make sure only a single Bitcoin process is using the data directory. fs::path pathLockFile = GetDataDir() / ".lock"; FILE *file = fopen(pathLockFile.string().c_str(), "a"); // empty lock file; created if it doesn't exist. @@ -1147,26 +1149,24 @@ bool AppInit2(thread_group &threadGroup) static boost::interprocess::file_lock lock(pathLockFile.string().c_str()); if (!lock.try_lock()) return InitError(strprintf( - _("Cannot obtain a lock on data directory %s. Eccoind is probably already running."), strDataDir)); + ("Cannot obtain a lock on data directory %s. Eccoind is probably already running."), strDataDir)); } catch (const boost::interprocess::interprocess_exception &e) { return InitError( - strprintf(_("Cannot obtain a lock on data directory %s. Eccoind is probably already running.") + " %s.", - strDataDir, e.what())); + strprintf("Cannot obtain a lock on data directory %s. Eccoind is probably already running. %s.", strDataDir, + e.what())); } #ifndef WIN32 CreatePidFile(GetPidFile(), getpid()); #endif - if (fPrintToDebugLog) - OpenDebugLog(); - - LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); - if (!fLogTimestamps) + if (!g_logger->fLogTimestamps) + { LogPrintf("Startup time: %s\n", DateTimeStrFormat("%Y-%m-%d %H:%M:%S", GetTime())); + } LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); LogPrintf("Using data directory %s\n", strDataDir); LogPrintf("Using config file %s\n", gArgs.GetConfigFile().string()); @@ -1190,9 +1190,8 @@ bool AppInit2(thread_group &threadGroup) */ if (fServer) { - uiInterface.InitMessage.connect(SetRPCWarmupStatus); if (!AppInitServers(threadGroup)) - return InitError(_("Unable to start HTTP server. See debug log for details.")); + return InitError(("Unable to start HTTP server. See debug log for details.")); } int64_t nStart; @@ -1201,7 +1200,7 @@ bool AppInit2(thread_group &threadGroup) LogPrintf("Using wallet %s\n", strWalletFile); - uiInterface.InitMessage(_("Verifying wallet...")); + LogPrintf("Verifying wallet..."); std::string warningString; std::string errorString; @@ -1229,14 +1228,14 @@ bool AppInit2(thread_group &threadGroup) for (auto cmt : gArgs.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) - return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); + return InitError(strprintf(("User Agent comment (%s) contains unsafe characters."), cmt)); uacomments.push_back(SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)); } strSubVersion = FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, uacomments); if (strSubVersion.size() > MAX_SUBVERSION_LENGTH) { - return InitError(strprintf(_("Total length of network version string (%i) exceeds maximum length (%i). Reduce " - "the number or size of uacomments."), + return InitError(strprintf(("Total length of network version string (%i) exceeds maximum length (%i). Reduce " + "the number or size of uacomments."), strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } @@ -1247,7 +1246,7 @@ bool AppInit2(thread_group &threadGroup) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) - return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); + return InitError(strprintf(("Unknown network specified in -onlynet: '%s'"), snet)); nets.insert(net); } for (int n = 0; n < NET_MAX; n++) @@ -1265,7 +1264,7 @@ bool AppInit2(thread_group &threadGroup) CSubNet subnet; LookupSubNet(net.c_str(), subnet); if (!subnet.IsValid()) - return InitError(strprintf(_("Invalid netmask specified in -whitelist: '%s'"), net)); + return InitError(strprintf(("Invalid netmask specified in -whitelist: '%s'"), net)); connman.AddWhitelistedRange(subnet); } } @@ -1282,7 +1281,7 @@ bool AppInit2(thread_group &threadGroup) proxyType addrProxy = proxyType(resolved, proxyRandomize); if (!addrProxy.IsValid()) { - return InitError(strprintf(_("Invalid -proxy address: '%s'"), proxyArg)); + return InitError(strprintf(("Invalid -proxy address: '%s'"), proxyArg)); } SetProxy(NET_IPV4, addrProxy); @@ -1311,7 +1310,7 @@ bool AppInit2(thread_group &threadGroup) proxyType addrOnion = proxyType(resolved, proxyRandomize); if (!addrOnion.IsValid()) { - return InitError(strprintf(_("Invalid -onion address: '%s'"), onionArg)); + return InitError(strprintf(("Invalid -onion address: '%s'"), onionArg)); } SetProxy(NET_TOR, addrOnion); SetLimited(NET_TOR, false); @@ -1333,16 +1332,16 @@ bool AppInit2(thread_group &threadGroup) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) - return InitError(strprintf(_("Cannot resolve -bind address: '%s'"), strBind)); + return InitError(strprintf(("Cannot resolve -bind address: '%s'"), strBind)); fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR)); } for (auto const &strBind : gArgs.GetArgs("-whitebind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, 0, false)) - return InitError(strprintf(_("Cannot resolve -whitebind address: '%s'"), strBind)); + return InitError(strprintf(("Cannot resolve -whitebind address: '%s'"), strBind)); if (addrBind.GetPort() == 0) - return InitError(strprintf(_("Need to specify a port with -whitebind: '%s'"), strBind)); + return InitError(strprintf(("Need to specify a port with -whitebind: '%s'"), strBind)); fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR | BF_WHITELIST)); } } @@ -1354,7 +1353,7 @@ bool AppInit2(thread_group &threadGroup) fBound |= Bind(connman, CService(inaddr_any, GetListenPort()), !fBound ? BF_REPORT_ERROR : BF_NONE); } if (!fBound) - return InitError(_("Failed to listen on any port. Use -listen=0 if you want this.")); + return InitError(("Failed to listen on any port. Use -listen=0 if you want this.")); } if (gArgs.IsArgSet("-externalip")) @@ -1467,14 +1466,14 @@ bool AppInit2(thread_group &threadGroup) // If necessary, upgrade from older database format. if (!pcoinsdbview->Upgrade()) { - strLoadError = _("Error upgrading chainstate database"); + strLoadError = ("Error upgrading chainstate database"); break; } } if (!pnetMan->getChainActive()->LoadBlockIndex()) { - strLoadError = _("Error loading block database"); + strLoadError = ("Error loading block database"); break; } // If the loaded chain has a wrong genesis, bail out immediately @@ -1485,14 +1484,14 @@ bool AppInit2(thread_group &threadGroup) pnetMan->getChainActive()->mapBlockIndex.count(chainparams.GetConsensus().hashGenesisBlock) == 0) { - return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); + return InitError("Incorrect or no genesis block found. Wrong datadir for network?"); } } // Initialize the block index (no-op if non-empty database was already loaded) if (!pnetMan->getChainActive()->InitBlockIndex(chainparams)) { - strLoadError = _("Error initializing block database"); + strLoadError = ("Error initializing block database"); break; } @@ -1518,10 +1517,10 @@ bool AppInit2(thread_group &threadGroup) CBlockIndex *tip = pnetMan->getChainActive()->chainActive.Tip(); if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { - strLoadError = _("The block database contains a block which appears to be from the future. " - "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's " - "date and time are correct"); + strLoadError = ("The block database contains a block which appears to be from the future. " + "This may be due to your computer's date and time being set incorrectly. " + "Only rebuild the block database if you are sure that your computer's " + "date and time are correct"); break; } } @@ -1530,15 +1529,15 @@ bool AppInit2(thread_group &threadGroup) gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { - strLoadError = _("Corrupted block database detected"); + strLoadError = ("Corrupted block database detected"); break; } } catch (const std::exception &e) { - if (fDebug) + if (g_logger->fDebug) LogPrintf("%s\n", e.what()); - strLoadError = _("Error opening block database"); + strLoadError = ("Error opening block database"); break; } @@ -1550,19 +1549,8 @@ bool AppInit2(thread_group &threadGroup) // first suggest a reindex if (!fReset) { - bool fRet = uiInterface.ThreadSafeMessageBox( - strLoadError + ".\n\n" + _("Do you want to rebuild the block database now?"), "", - CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); - if (fRet) - { - fReindex = true; - shutdown_threads.store(false); - } - else - { - LogPrintf("Aborted block database rebuild. Exiting.\n"); - return false; - } + LogPrintf("Aborted block database rebuild. Exiting.\n"); + return false; } else { @@ -1596,10 +1584,7 @@ bool AppInit2(thread_group &threadGroup) // ********************************************************* Step 10: import blocks - if (gArgs.IsArgSet("-blocknotify")) - uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); - - LogPrintf("Activating best chain..."); + LogPrintf("Activating best chain...\n"); std::vector vImportFiles; if (gArgs.IsArgSet("-loadblock")) diff --git a/src/init.h b/src/init.h index e408b000..c289c801 100644 --- a/src/init.h +++ b/src/init.h @@ -48,6 +48,8 @@ std::string HelpMessage(); /** Returns licensing information (for -version) */ std::string LicenseInfo(); +void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex); + extern std::unique_ptr pcoinsdbview; diff --git a/src/kernel.cpp b/src/kernel.cpp index 6f13a674..b921175a 100644 --- a/src/kernel.cpp +++ b/src/kernel.cpp @@ -32,6 +32,7 @@ #include "script/stakescript.h" #include "timedata.h" #include "txdb.h" +#include "util/logger.h" #include "util/utiltime.h" // The stake modifier used to hash for a stake kernel is chosen as the stake @@ -56,10 +57,7 @@ static bool GetKernelStakeModifier(uint256 hashBlockFrom, uint256 &nStakeModifie } if (blocksToGo > 0) { - if (fDebug) - { - LogPrintf("blocks to go was %i and it should be 0 but we ran out of indexes \n", blocksToGo); - } + LogPrint("kernel", "blocks to go was %i and it should be 0 but we ran out of indexes \n", blocksToGo); return false; } @@ -132,15 +130,15 @@ bool ComputeNextStakeModifier(const CBlockIndex *pindexPrev, const CTransaction CBlockIndex *index = pnetMan->getChainActive()->LookupBlockIndex(blockHashOfTx); if (!ReadBlockFromDisk(block, index, pnetMan->getActivePaymentNetwork()->GetConsensus())) + { // unable to read block of previous transaction - return fDebug ? error("ComputeNextStakeModifier() : read block failed") : false; + LogPrint("kernel", "ComputeNextStakeModifier() : read block failed"); + return false; + } if (!GetKernelStakeModifier(block.GetHash(), nStakeModifier)) { - if (fDebug) - { - LogPrintf("ComputeNextStakeModifier(): GetKernelStakeModifier return false\n"); - } + LogPrint("kernel", "ComputeNextStakeModifier(): GetKernelStakeModifier return false\n"); return false; } return true; @@ -194,10 +192,7 @@ bool CheckStakeKernelHash(int nHeight, if (nTimeWeight <= 0) { - if (fDebug) - { - LogPrintf("CheckStakeKernelHash(): ERROR: time weight was somehow <= 0 \n"); - } + LogPrint("kernel", "CheckStakeKernelHash(): ERROR: time weight was somehow <= 0 \n"); return false; } @@ -209,10 +204,7 @@ bool CheckStakeKernelHash(int nHeight, if (!GetKernelStakeModifier(blockFrom.GetHash(), nStakeModifier)) { - if (fDebug) - { - LogPrintf(">>> CheckStakeKernelHash: GetKernelStakeModifier return false\n"); - } + LogPrint("kernel", ">>> CheckStakeKernelHash: GetKernelStakeModifier return false\n"); return false; } // LogPrintf(">>> CheckStakeKernelHash: passed GetKernelStakeModifier\n"); @@ -248,38 +240,23 @@ bool CheckStakeKernelHash(int nHeight, std::string reductionHex = reduction.GetHex(); unsigned int n = std::count(reductionHex.begin(), reductionHex.end(), '0'); unsigned int redux = 64 - n; // 64 is max 0's in a 256 bit hex string - if (fDebug) - { - LogPrintf("reduction = %u \n", redux); - LogPrintf("pre reduction hashProofOfStake = %s \n", arith_hashProofOfStake.GetHex().c_str()); - } + LogPrint("kernel", "reduction = %u \n", redux); + LogPrint("kernel", "pre reduction hashProofOfStake = %s \n", arith_hashProofOfStake.GetHex().c_str()); // before we apply reduction, we want to shift the hash 20 bits to the right. the PoS limit is lead by 20 0's so // we want our reduction to apply to a hashproofofstake that is also lead by 20 0's arith_hashProofOfStake = arith_hashProofOfStake >> 20; - if (fDebug) - { - LogPrintf("mid reduction hashProofOfStake = %s \n", arith_hashProofOfStake.GetHex().c_str()); - } + LogPrint("kernel", "mid reduction hashProofOfStake = %s \n", arith_hashProofOfStake.GetHex().c_str()); arith_hashProofOfStake = arith_hashProofOfStake >> redux; - if (fDebug) - { - LogPrintf("post reduction hashProofOfStake = %s \n", arith_hashProofOfStake.GetHex().c_str()); - } + LogPrint("kernel", "post reduction hashProofOfStake = %s \n", arith_hashProofOfStake.GetHex().c_str()); // Now check if proof-of-stake hash meets target protocol if (arith_hashProofOfStake > hashTarget) { - if (fDebug) - { - LogPrintf("CheckStakeKernelHash(): ERROR: hashProofOfStake %s > %s hashTarget\n", - arith_hashProofOfStake.GetHex().c_str(), hashTarget.GetHex().c_str()); - } - return false; - } - if (fDebug) - { - LogPrintf("CheckStakeKernelHash(): SUCCESS: hashProofOfStake %s < %s hashTarget\n", + LogPrint("kernel", "CheckStakeKernelHash(): ERROR: hashProofOfStake %s > %s hashTarget\n", arith_hashProofOfStake.GetHex().c_str(), hashTarget.GetHex().c_str()); + return false; } + LogPrint("kernel", "CheckStakeKernelHash(): SUCCESS: hashProofOfStake %s < %s hashTarget\n", + arith_hashProofOfStake.GetHex().c_str(), hashTarget.GetHex().c_str()); } return true; @@ -309,8 +286,10 @@ bool CheckProofOfStake(int nHeight, const CTransaction &tx, uint256 &hashProofOf CBlockIndex *index = pnetMan->getChainActive()->LookupBlockIndex(blockHashOfTx); if (!ReadBlockFromDisk(block, index, pnetMan->getActivePaymentNetwork()->GetConsensus())) - // unable to read block of previous transaction - return fDebug ? error("CheckProofOfStake() : read block failed") : false; + { + LogPrint("kernel", "CheckProofOfStake() : read block failed"); + return false; + } CDiskTxPos txindex; pnetMan->getChainActive()->pblocktree->ReadTxIndex(txPrev.GetHash(), txindex); diff --git a/src/keystore.cpp b/src/keystore.cpp index 2fbdf7b6..ecdae4a8 100644 --- a/src/keystore.cpp +++ b/src/keystore.cpp @@ -22,6 +22,7 @@ #include "key.h" #include "pubkey.h" +#include "util/logger.h" #include "util/util.h" bool CKeyStore::AddKey(const CKey &key) { return AddKeyPubKey(key, key.GetPubKey()); } diff --git a/src/main.cpp b/src/main.cpp index 2e7424d3..194f6ea5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -49,7 +49,7 @@ #include "tinyformat.h" #include "txdb.h" #include "txmempool.h" -#include "ui_interface.h" + #include "undo.h" #include "util/util.h" #include "util/utilmoneystr.h" @@ -1013,10 +1013,7 @@ bool CheckTxInputs(const CTransaction &tx, CValidationState &state, const CCoins if (nStakeReward > GetProofOfStakeReward(tx.GetCoinAge(nCoinAge, true), nSpendHeight) + DEFAULT_TRANSACTION_MINFEE) { - if (fDebug) - { - LogPrintf("nStakeReward = %d , CoinAge = %d \n", nStakeReward, nCoinAge); - } + LogPrint("kernel", "nStakeReward = %d , CoinAge = %d \n", nStakeReward, nCoinAge); return state.DoS(100, false, REJECT_INVALID, "bad-txns-stake-reward-too-high", false, strprintf("ConnectInputs() : %s stake reward exceeded", tx.GetHash().ToString().substr(0, 10).c_str())); } @@ -1112,9 +1109,7 @@ bool AbortNode(const std::string &strMessage, const std::string &userMessage) { strMiscWarning = strMessage; LogPrintf("*** %s\n", strMessage); - uiInterface.ThreadSafeMessageBox( - userMessage.empty() ? _("Error: A fatal internal error occurred, see debug.log for details") : userMessage, "", - CClientUIInterface::MSG_ERROR); + LogPrintf("Error: A fatal internal error occurred, see debug.log for details\n"); StartShutdown(); return false; } @@ -1757,7 +1752,7 @@ bool CheckDiskSpace(uint64_t nAdditionalBytes) // Check for nMinDiskSpace bytes (currently 50MB) if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) - return AbortNode("Disk space is low!", _("Error: Disk space is low!")); + return AbortNode("Disk space is low!", "Error: Disk space is low!"); return true; } @@ -1805,42 +1800,33 @@ std::string GetWarnings(const std::string &strFor) { std::string strStatusBar; std::string strRPC; - std::string strGUI; if (!CLIENT_VERSION_IS_RELEASE) { strStatusBar = "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"; - strGUI = _( - "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); } if (gArgs.GetBoolArg("-testsafemode", DEFAULT_TESTSAFEMODE)) - strStatusBar = strRPC = strGUI = "testsafemode enabled"; + strStatusBar = strRPC = "testsafemode enabled"; // Misc warnings like out of disk space and clock is wrong if (strMiscWarning != "") { - strStatusBar = strGUI = strMiscWarning; + strStatusBar = strMiscWarning; } if (fLargeWorkForkFound) { strStatusBar = strRPC = "Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."; - strGUI = - _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); } else if (fLargeWorkInvalidChainFound) { strStatusBar = strRPC = "Warning: We do not appear to fully agree with our peers! You may need to upgrade, or " "other nodes may need to upgrade."; - strGUI = _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes " - "may need to upgrade."); } - if (strFor == "gui") - return strGUI; else if (strFor == "statusbar") return strStatusBar; else if (strFor == "rpc") @@ -1994,10 +1980,7 @@ int64_t GetProofOfStakeReward(int64_t nCoinAge, int nHeight) if (CMS == MAX_MONEY) { // if we are already at max money supply limits (25 billion coins, we return 0 as no new coins are to be minted - if (fDebug) - { - LogPrintf("GetProofOfStakeReward(): create=%i nCoinAge=%d\n", 0, nCoinAge); - } + LogPrint("kernel", "GetProofOfStakeReward(): create=%i nCoinAge=%d\n", 0, nCoinAge); return 0; } if (nHeight > 500000 && nHeight < 1005000) @@ -2013,10 +1996,7 @@ int64_t GetProofOfStakeReward(int64_t nCoinAge, int nHeight) nRewardCoinYear = 0; } int64_t nSubsidy = nCoinAge * nRewardCoinYear / 365; - if (fDebug) - { - LogPrintf("GetProofOfStakeReward(): create=%s nCoinAge=%d\n", FormatMoney(nSubsidy).c_str(), nCoinAge); - } + LogPrint("kernel", "GetProofOfStakeReward(): create=%s nCoinAge=%d\n", FormatMoney(nSubsidy).c_str(), nCoinAge); return nSubsidy; } @@ -2036,9 +2016,6 @@ int64_t GetProofOfStakeReward(int64_t nCoinAge, int nHeight) nSubsidy = nSubsidy - difference; } } - if (fDebug) - { - LogPrintf("GetProofOfStakeReward(): create=%s nCoinAge=%d\n", FormatMoney(nSubsidy).c_str(), nCoinAge); - } + LogPrint("kernel", "GetProofOfStakeReward(): create=%s nCoinAge=%d\n", FormatMoney(nSubsidy).c_str(), nCoinAge); return nSubsidy; } diff --git a/src/net/addrman.h b/src/net/addrman.h index fc1663a6..2e33bd0b 100644 --- a/src/net/addrman.h +++ b/src/net/addrman.h @@ -24,6 +24,7 @@ #include "random.h" #include "sync.h" #include "timedata.h" +#include "util/logger.h" #include "util/util.h" #include diff --git a/src/net/messages.cpp b/src/net/messages.cpp index 12bb745b..e20c1d9c 100644 --- a/src/net/messages.cpp +++ b/src/net/messages.cpp @@ -112,7 +112,7 @@ void PushNodeVersion(CNode *pnode, CConnman &connman, int64_t nTime) connman.PushMessage(pnode, NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, nLocalHostNonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes); - if (fLogIPs) + if (g_logger->fLogIPs) { LogPrintf("send version message: version %d, blocks=%d, " "us=%s, them=%s, peer=%d\n", @@ -1173,7 +1173,7 @@ bool static ProcessMessage(CNode *pfrom, } std::string remoteAddr; - if (fLogIPs) + if (g_logger->fLogIPs) { remoteAddr = ", peeraddr=" + pfrom->addr.ToString(); } @@ -2111,7 +2111,7 @@ bool static ProcessMessage(CNode *pfrom, else if (strCommand == NetMsgType::REJECT) { - if (fDebug) + if (g_logger->fDebug) { try { diff --git a/src/net/net.cpp b/src/net/net.cpp index c1382c72..48d4c486 100644 --- a/src/net/net.cpp +++ b/src/net/net.cpp @@ -29,7 +29,7 @@ #include "init.h" #include "net/addrman.h" #include "networks/netman.h" -#include "ui_interface.h" + #include "util/utilstrencodings.h" #ifdef WIN32 @@ -598,7 +598,7 @@ void CConnman::SweepBanned() { setBanned.erase(it++); setBannedIsDirty = true; - LogPrint("%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); + LogPrintf("%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); } else { @@ -1634,7 +1634,7 @@ void MapPort(bool fUseUPnP) delete upnp_thread; upnp_thread = nullptr; } - upnp_thread = new std::thread(&TraceThread, "upnp", &ThreadMapPort); + upnp_thread = new std::thread(&ThreadMapPort); } else if (upnp_thread) { @@ -2313,14 +2313,14 @@ bool CConnman::BindListenPort(const CService &addrBind, std::string &strError, b int nErr = WSAGetLastError(); if (nErr == WSAEADDRINUSE) { - strError = strprintf(_("Unable to bind to %s on this computer. %s " - "is probably already running."), - addrBind.ToString(), _("Eccoind")); + strError = strprintf("Unable to bind to %s on this computer. %s " + "is probably already running.", + addrBind.ToString(), "Eccoind"); } else { - strError = strprintf(_("Unable to bind to %s on this computer " - "(bind returned error %s)"), + strError = strprintf("Unable to bind to %s on this computer " + "(bind returned error %s)", addrBind.ToString(), NetworkErrorString(nErr)); } LogPrintf("%s\n", strError); @@ -2332,8 +2332,8 @@ bool CConnman::BindListenPort(const CService &addrBind, std::string &strError, b // Listen for incoming connections if (listen(hListenSocket, SOMAXCONN) == SOCKET_ERROR) { - strError = strprintf(_("Error: Listening for incoming connections " - "failed (listen returned error %s)"), + strError = strprintf("Error: Listening for incoming connections " + "failed (listen returned error %s)", NetworkErrorString(WSAGetLastError())); LogPrintf("%s\n", strError); CloseSocket(hListenSocket); @@ -2946,7 +2946,7 @@ CNode::CNode(NodeId idIn, } mapRecvBytesPerMsgCmd[NET_MESSAGE_COMMAND_OTHER] = 0; - if (fLogIPs) + if (g_logger->fLogIPs) { LogPrintf("Added connection to %s peer=%d\n", addrName, id); } diff --git a/src/net/netbase.cpp b/src/net/netbase.cpp index 51cec03c..4309bde6 100644 --- a/src/net/netbase.cpp +++ b/src/net/netbase.cpp @@ -30,6 +30,7 @@ #include "random.h" #include "sync.h" #include "uint256.h" +#include "util/logger.h" #include "util/util.h" #include "util/utilstrencodings.h" diff --git a/src/net/protocol.cpp b/src/net/protocol.cpp index e00e0052..692011ef 100644 --- a/src/net/protocol.cpp +++ b/src/net/protocol.cpp @@ -20,6 +20,7 @@ #include "net/protocol.h" +#include "util/logger.h" #include "util/util.h" #include "util/utilstrencodings.h" diff --git a/src/networks/netman.cpp b/src/networks/netman.cpp index 15061684..00b604bb 100644 --- a/src/networks/netman.cpp +++ b/src/networks/netman.cpp @@ -28,8 +28,8 @@ void AppendParamsHelpMessages(std::string &strUsage, bool debugHelp) { - strUsage += HelpMessageGroup(_("Chain selection options:")); - strUsage += HelpMessageOpt("-testnet", _("Use the test chain")); + strUsage += HelpMessageGroup("Chain selection options:"); + strUsage += HelpMessageOpt("-testnet", "Use the test chain"); if (debugHelp) { strUsage += HelpMessageOpt("-regtest", diff --git a/src/noui.cpp b/src/noui.cpp deleted file mode 100644 index b7712c6f..00000000 --- a/src/noui.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2009-2010 Satoshi Nakamoto - * Copyright (c) 2009-2016 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - - -#include "noui.h" - -#include "ui_interface.h" -#include "util/util.h" - -#include -#include -#include - -static bool noui_ThreadSafeMessageBox(const std::string &message, const std::string &caption, unsigned int style) -{ - bool fSecure = style & CClientUIInterface::SECURE; - style &= ~CClientUIInterface::SECURE; - - std::string strCaption; - // Check for usage of predefined caption - switch (style) - { - case CClientUIInterface::MSG_ERROR: - strCaption += _("Error"); - break; - case CClientUIInterface::MSG_WARNING: - strCaption += _("Warning"); - break; - case CClientUIInterface::MSG_INFORMATION: - strCaption += _("Information"); - break; - default: - strCaption += caption; // Use supplied caption (can be empty) - } - - if (!fSecure) - LogPrintf("%s: %s\n", strCaption, message); - fprintf(stderr, "%s: %s\n", strCaption.c_str(), message.c_str()); - return false; -} - -static void noui_InitMessage(const std::string &message) { LogPrintf("init message: %s\n", message); } -void noui_connect() -{ - // Connect bitcoind signal handlers - uiInterface.ThreadSafeMessageBox.connect(noui_ThreadSafeMessageBox); - uiInterface.InitMessage.connect(noui_InitMessage); -} diff --git a/src/noui.h b/src/noui.h deleted file mode 100644 index d10765ea..00000000 --- a/src/noui.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2009-2017 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef BITCOIN_NOUI_H -#define BITCOIN_NOUI_H - -extern void noui_connect(); - -#endif // BITCOIN_NOUI_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index cc7a7bce..d781797c 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -26,6 +26,7 @@ #include "chain/tx.h" #include "streams.h" #include "txmempool.h" +#include "util/logger.h" #include "util/util.h" void TxConfirmStats::Initialize(std::vector &defaultBuckets, diff --git a/src/processblock.cpp b/src/processblock.cpp index 85f2f6a5..179d50b8 100644 --- a/src/processblock.cpp +++ b/src/processblock.cpp @@ -43,7 +43,7 @@ #include "processheader.h" #include "processtx.h" #include "txmempool.h" -#include "ui_interface.h" + #include "undo.h" #include "util/util.h" #include "validationinterface.h" @@ -235,7 +235,7 @@ void UpdateTip(CBlockIndex *pindexNew) { if (state == THRESHOLD_ACTIVE) { - strMiscWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); + strMiscWarning = strprintf("Warning: unknown new rules activated (versionbit %i)", bit); if (!fWarned) { fWarned = true; @@ -259,8 +259,7 @@ void UpdateTip(CBlockIndex *pindexNew) if (nUpgraded > 100 / 2) { // strMiscWarning is read by GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - strMiscWarning = - _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); + strMiscWarning = "Warning: Unknown block versions being mined! It's possible unknown rules are in effect"; if (!fWarned) { fWarned = true; @@ -641,7 +640,7 @@ bool ActivateBestChain(CValidationState &state, const CNetworkTemplate &chainpar // Always notify the UI if a new block tip was connected if (pindexFork != pindexNewTip) { - uiInterface.NotifyBlockTip(fInitialDownload, pindexNewTip); + BlockNotifyCallback(fInitialDownload, pindexNewTip); if (!fInitialDownload) { diff --git a/src/random.cpp b/src/random.cpp index 423cc2b7..0d351638 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -27,7 +27,8 @@ #include #endif #include "serialize.h" -#include "util/util.h" // for LogPrint() +#include "util/logger.h" +#include "util/util.h" #include "util/utilstrencodings.h" // for GetTime() #include #include diff --git a/src/rpc/rpcdump.cpp b/src/rpc/rpcdump.cpp index 6cce3962..6d5b700a 100644 --- a/src/rpc/rpcdump.cpp +++ b/src/rpc/rpcdump.cpp @@ -343,7 +343,7 @@ UniValue importwallet(const UniValue ¶ms, bool fHelp) int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); file.seekg(0, file.beg); - pwalletMain->ShowProgress(_("Importing..."), 0); // show progress dialog in GUI + pwalletMain->ShowProgress("Importing...", 0); // show progress dialog in GUI while (file.good()) { pwalletMain->ShowProgress( diff --git a/src/rpc/rpcnet.cpp b/src/rpc/rpcnet.cpp index 1972cbef..ed088c0b 100644 --- a/src/rpc/rpcnet.cpp +++ b/src/rpc/rpcnet.cpp @@ -29,7 +29,7 @@ #include "networks/netman.h" #include "sync.h" #include "timedata.h" -#include "ui_interface.h" + #include "util/util.h" #include "util/utilstrencodings.h" #include "version.h" diff --git a/src/rpc/rpcprotocol.cpp b/src/rpc/rpcprotocol.cpp index f8ffd917..e25c5ce7 100644 --- a/src/rpc/rpcprotocol.cpp +++ b/src/rpc/rpcprotocol.cpp @@ -23,6 +23,7 @@ #include "args.h" #include "random.h" #include "tinyformat.h" +#include "util/logger.h" #include "util/util.h" #include "util/utilstrencodings.h" #include "util/utiltime.h" diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 31f8399c..4949d412 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -27,7 +27,7 @@ #include "init.h" #include "random.h" #include "sync.h" -#include "ui_interface.h" + #include "util/util.h" #include "util/utilstrencodings.h" @@ -758,8 +758,8 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string &strMethod, co if (!GetAuthCookie(&strRPCUserColonPass)) { throw std::runtime_error( - strprintf(_("Could not locate RPC credentials. No authentication cookie could be found, and RPC " - "password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)"), + strprintf("Could not locate RPC credentials. No authentication cookie could be found, and RPC " + "password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)", gArgs.GetConfigFile().string().c_str())); } } diff --git a/src/sync.cpp b/src/sync.cpp index 8f8d3ab2..18a6e019 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -20,6 +20,7 @@ #include "sync.h" +#include "util/logger.h" #include "util/util.h" #include "util/utilstrencodings.h" diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index c51c2f1b..592a95e4 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -23,29 +23,25 @@ #include "test/testutil.h" #include "txdb.h" #include "txmempool.h" -#include "ui_interface.h" + +#include "util/logger.h" #include #include #include - -extern bool fPrintToConsole; -extern void noui_connect(); CWallet *pwallet = nullptr; - BasicTestingSetup::BasicTestingSetup(const std::string &chainName) { ECC_Start(); SetupEnvironment(); SetupNetworking(); - fPrintToDebugLog = false; // don't want to write to debug.log file + g_logger->fPrintToDebugLog = false; // don't want to write to debug.log file fCheckBlockIndex = true; pnetMan = new CNetworkManager(); pwallet = new CWallet("walletFile"); pnetMan->SetParams(chainName); - noui_connect(); } BasicTestingSetup::~BasicTestingSetup() { ECC_Stop(); } @@ -161,13 +157,13 @@ struct StartupShutdown std::string s = opts["log_bitcoin"].as(); if (s == "console") { - fPrintToConsole = true; - fPrintToDebugLog = false; + g_logger->fPrintToConsole = true; + g_logger->fPrintToDebugLog = false; } else if (s == "none") { - fPrintToConsole = false; - fPrintToDebugLog = false; + g_logger->fPrintToConsole = false; + g_logger->fPrintToDebugLog = false; } } } diff --git a/src/timedata.cpp b/src/timedata.cpp index def86da7..bdf39356 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -21,7 +21,8 @@ #include "net/netbase.h" #include "sync.h" -#include "ui_interface.h" + +#include "util/logger.h" #include "util/util.h" #include "util/utilstrencodings.h" @@ -103,10 +104,10 @@ void AddTimeData(const CNetAddr &ip, int64_t nOffsetSample) if (!fMatch) { fDone = true; - std::string strMessage = _("Please check that your computer's date and time are correct! If your " - "clock is wrong Bitcoin Core will not work properly."); + std::string strMessage = "Please check that your computer's date and time are correct! If your " + "clock is wrong Bitcoin Core will not work properly."; strMiscWarning = strMessage; - uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_WARNING); + LogPrintf("%s\n", strMessage.c_str()); } } } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index ae24f521..7102b72c 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -777,7 +777,7 @@ void StartTorControl(thread_group &threadGroup) return; } - torControlThread = std::thread(boost::bind(&TraceThread, "torcontrol", &TorControlThread)); + torControlThread = std::thread(&TorControlThread); } void InterruptTorControl() diff --git a/src/ui_interface.h b/src/ui_interface.h deleted file mode 100644 index 0c4b841d..00000000 --- a/src/ui_interface.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2009-2010 Satoshi Nakamoto - * Copyright (c) 2009-2016 The Bitcoin Core developers - * Copyright (c) 2014-2018 The Eccoin developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef BITCOIN_UI_INTERFACE_H -#define BITCOIN_UI_INTERFACE_H - -#include -#include - -#include -#include - -class CBasicKeyStore; -class CWallet; -class uint256; -class CBlockIndex; - -/** General change type (added, updated, removed). */ -enum ChangeType -{ - CT_NEW, - CT_UPDATED, - CT_DELETED -}; - -/** Signals for UI communication. */ -class CClientUIInterface -{ -public: - /** Flags for CClientUIInterface::ThreadSafeMessageBox */ - enum MessageBoxFlags - { - ICON_INFORMATION = 0, - ICON_WARNING = (1U << 0), - ICON_ERROR = (1U << 1), - /** - * Mask of all available icons in CClientUIInterface::MessageBoxFlags - * This needs to be updated, when icons are changed there! - */ - ICON_MASK = (ICON_INFORMATION | ICON_WARNING | ICON_ERROR), - - /** These values are taken from qmessagebox.h "enum StandardButton" to be directly usable */ - BTN_OK = 0x00000400U, // QMessageBox::Ok - BTN_YES = 0x00004000U, // QMessageBox::Yes - BTN_NO = 0x00010000U, // QMessageBox::No - BTN_ABORT = 0x00040000U, // QMessageBox::Abort - BTN_RETRY = 0x00080000U, // QMessageBox::Retry - BTN_IGNORE = 0x00100000U, // QMessageBox::Ignore - BTN_CLOSE = 0x00200000U, // QMessageBox::Close - BTN_CANCEL = 0x00400000U, // QMessageBox::Cancel - BTN_DISCARD = 0x00800000U, // QMessageBox::Discard - BTN_HELP = 0x01000000U, // QMessageBox::Help - BTN_APPLY = 0x02000000U, // QMessageBox::Apply - BTN_RESET = 0x04000000U, // QMessageBox::Reset - /** - * Mask of all available buttons in CClientUIInterface::MessageBoxFlags - * This needs to be updated, when buttons are changed there! - */ - BTN_MASK = - (BTN_OK | BTN_YES | BTN_NO | BTN_ABORT | BTN_RETRY | BTN_IGNORE | BTN_CLOSE | BTN_CANCEL | BTN_DISCARD | - BTN_HELP | - BTN_APPLY | - BTN_RESET), - - /** Force blocking, modal message box dialog (not just OS notification) */ - MODAL = 0x10000000U, - - /** Do not print contents of message to debug log */ - SECURE = 0x40000000U, - - /** Predefined combinations for certain default usage cases */ - MSG_INFORMATION = ICON_INFORMATION, - MSG_WARNING = (ICON_WARNING | BTN_OK | MODAL), - MSG_ERROR = (ICON_ERROR | BTN_OK | MODAL) - }; - - /** Show message box. */ - boost::signals2::signal > - ThreadSafeMessageBox; - - /** Progress message during initialization. */ - boost::signals2::signal InitMessage; - - /** Number of network connections changed. */ - boost::signals2::signal NotifyNumConnectionsChanged; - - /** A wallet has been loaded. */ - boost::signals2::signal LoadWallet; - - /** Show progress e.g. for verifychain */ - boost::signals2::signal ShowProgress; - - /** New block has been accepted */ - boost::signals2::signal NotifyBlockTip; - - /** Banlist did change. */ - boost::signals2::signal BannedListChanged; -}; - -extern CClientUIInterface uiInterface; - -#endif // BITCOIN_UI_INTERFACE_H diff --git a/src/util/logger.cpp b/src/util/logger.cpp new file mode 100644 index 00000000..e987e979 --- /dev/null +++ b/src/util/logger.cpp @@ -0,0 +1,212 @@ + +#include "args.h" +#include "logger.h" +#include "fs.h" +#include "serialize.h" +#include "util.h" +#include "utiltime.h" +#include + +#include + +CLogger* g_logger = nullptr; +volatile bool fReopenDebugLog = false; + +bool CLogger::DebugPrintInit() +{ + if (mutexDebugLog == nullptr) + { + mutexDebugLog = new std::mutex(); + } + if (vMsgsBeforeOpenLog == nullptr) + { + vMsgsBeforeOpenLog = new std::list; + } + return true; +} + +/** + * fStartedNewLine is a state variable held by the calling context that will + * suppress printing of the timestamp when multiple calls are made that don't + * end in a newline. Initialize it to true, and hold it, in the calling context. + */ +std::string CLogger::LogTimestampStr(const std::string &str, bool *fStartedNewLine) +{ + std::string strStamped; + + if (!fLogTimestamps) + return str; + + if (*fStartedNewLine) + { + int64_t nTimeMicros = GetLogTimeMicros(); + strStamped = DateTimeStrFormat("%Y-%m-%d %H:%M:%S", nTimeMicros / 1000000); + if (fLogTimeMicros) + strStamped += strprintf(".%06d", nTimeMicros % 1000000); + strStamped += ' ' + str; + } + else + strStamped = str; + + if (!str.empty() && str[str.size() - 1] == '\n') + *fStartedNewLine = true; + else + *fStartedNewLine = false; + + return strStamped; +} + +int CLogger::FileWriteStr(const std::string &str, FILE *fp) +{ + return fwrite(str.data(), 1, str.size(), fp); +} + +void CLogger::OpenDebugLog() +{ + if (!DebugPrintInit()) + { + return; + } + std::lock_guard scoped_lock(*mutexDebugLog); + + assert(fileout == nullptr); + assert(vMsgsBeforeOpenLog); + fs::path pathDebug = GetDataDir() / "debug.log"; + fileout = fopen(pathDebug.string().c_str(), "a"); + if (fileout) + setbuf(fileout, NULL); // unbuffered + + // dump buffered messages from before we opened the log + while (!vMsgsBeforeOpenLog->empty()) + { + FileWriteStr(vMsgsBeforeOpenLog->front(), fileout); + vMsgsBeforeOpenLog->pop_front(); + } + + delete vMsgsBeforeOpenLog; + vMsgsBeforeOpenLog = nullptr; +} + +bool CLogger::LogAcceptCategory(const char *category) +{ + if (category != nullptr) + { + if (!fDebug) + return false; + + // Give each thread quick access to -debug settings. + // This helps prevent issues debugging global destructors, + // where mapMultiArgs might be deleted before another + // global destructor calls LogPrint() + static boost::thread_specific_ptr > ptrCategory; + if (ptrCategory.get() == NULL) + { + const std::vector &categories = gArgs.GetArgs("-debug"); + ptrCategory.reset(new std::set(categories.begin(), categories.end())); + // thread_specific_ptr automatically deletes the set when the thread ends. + } + const std::set &setCategories = *ptrCategory.get(); + + // if not debugging everything and not debugging specific category, LogPrint does nothing. + if (setCategories.count(std::string("")) == 0 && setCategories.count(std::string("1")) == 0 && + setCategories.count(std::string(category)) == 0) + return false; + } + return true; +} + +int CLogger::LogPrintStr(const std::string &str) +{ + int ret = 0; // Returns total number of characters written + static bool fStartedNewLine = true; + + std::string strTimestamped = LogTimestampStr(str, &fStartedNewLine); + + if (fPrintToConsole) + { + // print to console + ret = fwrite(strTimestamped.data(), 1, strTimestamped.size(), stdout); + fflush(stdout); + } + else if (fPrintToDebugLog) + { + std::lock_guard scoped_lock(*mutexDebugLog); + + // buffer if we haven't opened the log yet + if (fileout == nullptr) + { + assert(vMsgsBeforeOpenLog); + ret = strTimestamped.length(); + vMsgsBeforeOpenLog->push_back(strTimestamped); + } + else + { + // reopen the log file, if requested + if (fReopenDebugLog) + { + fReopenDebugLog = false; + fs::path pathDebug = GetDataDir() / "debug.log"; + if (freopen(pathDebug.string().c_str(), "a", fileout) != NULL) + setbuf(fileout, NULL); // unbuffered + } + + ret = FileWriteStr(strTimestamped, fileout); + } + } + return ret; +} + +void CLogger::ShrinkDebugFile() +{ + // Scroll debug.log if it's getting too big + fs::path pathLog = GetDataDir() / "debug.log"; + FILE *file = fopen(pathLog.string().c_str(), "r"); + if (file && fs::file_size(pathLog) > 20000) + { + // Restart the file with some of the end + std::vector vch(200000, 0); + fseek(file, -((long)vch.size()), SEEK_END); + int nBytes = fread(begin_ptr(vch), 1, vch.size(), file); + fclose(file); + + file = fopen(pathLog.string().c_str(), "w"); + if (file) + { + fwrite(begin_ptr(vch), 1, nBytes, file); + fclose(file); + } + } + else if (file != nullptr) + fclose(file); +} + +static std::string FormatException(const std::exception *pex, const char *pszThread) +{ +#ifdef WIN32 + char pszModule[MAX_PATH] = ""; + GetModuleFileNameA(NULL, pszModule, sizeof(pszModule)); +#else + const char *pszModule = "eccoin"; +#endif + if (pex) + return strprintf("EXCEPTION: %s \n%s \n%s in %s \n", typeid(*pex).name(), pex->what(), + pszModule, pszThread); + else + return strprintf("UNKNOWN EXCEPTION \n%s in %s \n", pszModule, pszThread); +} + +void PrintException(const std::exception *pex, const char *pszThread) +{ + std::string message = FormatException(pex, pszThread); + LogPrintf("\n\n************************\n%s\n", message); + fprintf(stderr, "\n\n************************\n%s\n", message.c_str()); + throw; +} + + +void PrintExceptionContinue(const std::exception *pex, const char *pszThread) +{ + std::string message = FormatException(pex, pszThread); + LogPrintf("\n\n************************\n%s\n", message); + fprintf(stderr, "\n\n************************\n%s\n", message.c_str()); +} diff --git a/src/util/logger.h b/src/util/logger.h new file mode 100644 index 00000000..6b88efd1 --- /dev/null +++ b/src/util/logger.h @@ -0,0 +1,118 @@ + + +#ifndef ECCOIN_LOGGER_H +#define ECCOIN_LOGGER_H + +#include "tinyformat.h" + +#include +#include +#include + +static const bool DEFAULT_LOGTIMEMICROS = false; +static const bool DEFAULT_LOGIPS = false; +static const bool DEFAULT_LOGTIMESTAMPS = true; +extern volatile bool fReopenDebugLog; + +class CLogger +{ +private: + FILE *fileout; + std::mutex *mutexDebugLog; + std::list *vMsgsBeforeOpenLog; + +public: + bool fLogTimestamps; + bool fLogTimeMicros; + bool fLogIPs; + + bool fDebug; + bool fPrintToConsole; + bool fPrintToDebugLog; + +private: + // Disallow copies + CLogger(const CLogger &); + CLogger &operator=(const CLogger &); + + bool DebugPrintInit(); + + std::string LogTimestampStr(const std::string &str, bool *fStartedNewLine); + + int FileWriteStr(const std::string &str, FILE *fp); + +public: + CLogger() + { + fileout = nullptr; + mutexDebugLog = nullptr; + vMsgsBeforeOpenLog = nullptr; + + fLogTimestamps = DEFAULT_LOGTIMESTAMPS; + fLogTimeMicros = DEFAULT_LOGTIMEMICROS; + fLogIPs = DEFAULT_LOGIPS; + fDebug = false; + fPrintToConsole = false; + fPrintToDebugLog = true; + DebugPrintInit(); + } + void OpenDebugLog(); + + /** Return true if log accepts specified category */ + bool LogAcceptCategory(const char *category); + + /** Send a string to the log output */ + int LogPrintStr(const std::string &str); + + void ShrinkDebugFile(); +}; + +extern CLogger* g_logger; + +#define LogPrintf(...) LogPrint(NULL, __VA_ARGS__) + +/** + * Zero-arg versions of logging and error, these are not covered by + * TINYFORMAT_FOREACH_ARGNUM + */ +static inline int LogPrint(const char *category, const char *format) +{ + if (!g_logger->LogAcceptCategory(category)) + return 0; + return g_logger->LogPrintStr(format); +} +static inline bool error(const char *format) +{ + g_logger->LogPrintStr(std::string("ERROR: ") + format + "\n"); + return false; +} + +void PrintException(const std::exception *pex, const char *pszThread); +void PrintExceptionContinue(const std::exception *pex, const char *pszThread); + + +/** + * When we switch to C++11, this can be switched to variadic templates instead + * of this macro-based construction (see tinyformat.h). + */ +#define MAKE_ERROR_AND_LOG_FUNC(n) \ + /** Print to debug.log if -debug=category switch is given OR category is NULL. */ \ + template \ + static inline int LogPrint(const char *category, const char *format, TINYFORMAT_VARARGS(n)) \ + { \ + if (!g_logger->LogAcceptCategory(category)) \ + return 0; \ + return g_logger->LogPrintStr(tfm::format(format, TINYFORMAT_PASSARGS(n))); \ + } \ + /** Log error and return false */ \ + template \ + static inline bool error(const char *format, TINYFORMAT_VARARGS(n)) \ + { \ + g_logger->LogPrintStr("ERROR: " + tfm::format(format, TINYFORMAT_PASSARGS(n)) + "\n"); \ + return false; \ + } + +TINYFORMAT_FOREACH_ARGNUM(MAKE_ERROR_AND_LOG_FUNC) + + +#endif diff --git a/src/util/util.cpp b/src/util/util.cpp index 548dd5d0..087ae80f 100644 --- a/src/util/util.cpp +++ b/src/util/util.cpp @@ -26,6 +26,7 @@ #include "random.h" #include "serialize.h" #include "sync.h" +#include "util/logger.h" #include "util/utilstrencodings.h" #include "util/utiltime.h" @@ -113,17 +114,9 @@ std::string to_internal(const std::string &); const char *const CONF_FILENAME = "eccoin.conf"; const char *const PID_FILENAME = "eccoind.pid"; -bool fDebug = false; -bool fPrintToConsole = false; -bool fPrintToDebugLog = true; bool fDaemon = false; bool fServer = false; std::string strMiscWarning; -bool fLogTimestamps = DEFAULT_LOGTIMESTAMPS; -bool fLogTimeMicros = DEFAULT_LOGTIMEMICROS; -bool fLogIPs = DEFAULT_LOGIPS; -volatile bool fReopenDebugLog = false; -CTranslationInterface translationInterface; /** Init OpenSSL library multithreading support */ static CCriticalSection **ppmutexOpenSSL; @@ -178,164 +171,6 @@ class CInit } } instance_of_cinit; -/** - * LogPrintf() has been broken a couple of times now - * by well-meaning people adding mutexes in the most straightforward way. - * It breaks because it may be called by global destructors during shutdown. - * Since the order of destruction of static/global objects is undefined, - * defining a mutex as a global object doesn't work (the mutex gets - * destroyed, and then some later destructor calls OutputDebugStringF, - * maybe indirectly, and you get a core dump at shutdown trying to lock - * the mutex). - */ - -static boost::once_flag debugPrintInitFlag = BOOST_ONCE_INIT; - -/** - * We use boost::call_once() to make sure mutexDebugLog and - * vMsgsBeforeOpenLog are initialized in a thread-safe manner. - * - * NOTE: fileout, mutexDebugLog and sometimes vMsgsBeforeOpenLog - * are leaked on exit. This is ugly, but will be cleaned up by - * the OS/libc. When the shutdown sequence is fully audited and - * tested, explicit destruction of these objects can be implemented. - */ -static FILE *fileout = NULL; -static boost::mutex *mutexDebugLog = NULL; -static std::list *vMsgsBeforeOpenLog; - -static int FileWriteStr(const std::string &str, FILE *fp) { return fwrite(str.data(), 1, str.size(), fp); } -static void DebugPrintInit() -{ - assert(mutexDebugLog == NULL); - mutexDebugLog = new boost::mutex(); - vMsgsBeforeOpenLog = new std::list; -} - -void OpenDebugLog() -{ - boost::call_once(&DebugPrintInit, debugPrintInitFlag); - boost::mutex::scoped_lock scoped_lock(*mutexDebugLog); - - assert(fileout == NULL); - assert(vMsgsBeforeOpenLog); - fs::path pathDebug = GetDataDir() / "debug.log"; - fileout = fopen(pathDebug.string().c_str(), "a"); - if (fileout) - setbuf(fileout, NULL); // unbuffered - - // dump buffered messages from before we opened the log - while (!vMsgsBeforeOpenLog->empty()) - { - FileWriteStr(vMsgsBeforeOpenLog->front(), fileout); - vMsgsBeforeOpenLog->pop_front(); - } - - delete vMsgsBeforeOpenLog; - vMsgsBeforeOpenLog = NULL; -} - -bool LogAcceptCategory(const char *category) -{ - if (category != NULL) - { - if (!fDebug) - return false; - - // Give each thread quick access to -debug settings. - // This helps prevent issues debugging global destructors, - // where mapMultiArgs might be deleted before another - // global destructor calls LogPrint() - static boost::thread_specific_ptr > ptrCategory; - if (ptrCategory.get() == NULL) - { - const std::vector &categories = gArgs.GetArgs("-debug"); - ptrCategory.reset(new std::set(categories.begin(), categories.end())); - // thread_specific_ptr automatically deletes the set when the thread ends. - } - const std::set &setCategories = *ptrCategory.get(); - - // if not debugging everything and not debugging specific category, LogPrint does nothing. - if (setCategories.count(std::string("")) == 0 && setCategories.count(std::string("1")) == 0 && - setCategories.count(std::string(category)) == 0) - return false; - } - return true; -} - -/** - * fStartedNewLine is a state variable held by the calling context that will - * suppress printing of the timestamp when multiple calls are made that don't - * end in a newline. Initialize it to true, and hold it, in the calling context. - */ -static std::string LogTimestampStr(const std::string &str, bool *fStartedNewLine) -{ - std::string strStamped; - - if (!fLogTimestamps) - return str; - - if (*fStartedNewLine) - { - int64_t nTimeMicros = GetLogTimeMicros(); - strStamped = DateTimeStrFormat("%Y-%m-%d %H:%M:%S", nTimeMicros / 1000000); - if (fLogTimeMicros) - strStamped += strprintf(".%06d", nTimeMicros % 1000000); - strStamped += ' ' + str; - } - else - strStamped = str; - - if (!str.empty() && str[str.size() - 1] == '\n') - *fStartedNewLine = true; - else - *fStartedNewLine = false; - - return strStamped; -} - -int LogPrintStr(const std::string &str) -{ - int ret = 0; // Returns total number of characters written - static bool fStartedNewLine = true; - - std::string strTimestamped = LogTimestampStr(str, &fStartedNewLine); - - if (fPrintToConsole) - { - // print to console - ret = fwrite(strTimestamped.data(), 1, strTimestamped.size(), stdout); - fflush(stdout); - } - else if (fPrintToDebugLog) - { - boost::call_once(&DebugPrintInit, debugPrintInitFlag); - boost::mutex::scoped_lock scoped_lock(*mutexDebugLog); - - // buffer if we haven't opened the log yet - if (fileout == NULL) - { - assert(vMsgsBeforeOpenLog); - ret = strTimestamped.length(); - vMsgsBeforeOpenLog->push_back(strTimestamped); - } - else - { - // reopen the log file, if requested - if (fReopenDebugLog) - { - fReopenDebugLog = false; - fs::path pathDebug = GetDataDir() / "debug.log"; - if (freopen(pathDebug.string().c_str(), "a", fileout) != NULL) - setbuf(fileout, NULL); // unbuffered - } - - ret = FileWriteStr(strTimestamped, fileout); - } - } - return ret; -} - static const int screenWidth = 79; static const int optIndent = 2; static const int msgIndent = 7; @@ -347,37 +182,6 @@ std::string HelpMessageOpt(const std::string &option, const std::string &message FormatParagraph(message, screenWidth - msgIndent, msgIndent) + std::string("\n\n"); } -static std::string FormatException(const std::exception *pex, const char *pszThread) -{ -#ifdef WIN32 - char pszModule[MAX_PATH] = ""; - GetModuleFileNameA(NULL, pszModule, sizeof(pszModule)); -#else - const char *pszModule = "eccoin"; -#endif - if (pex) - return strprintf("EXCEPTION: %s \n%s \n%s in %s \n", typeid(*pex).name(), pex->what(), - pszModule, pszThread); - else - return strprintf("UNKNOWN EXCEPTION \n%s in %s \n", pszModule, pszThread); -} - -void PrintException(const std::exception *pex, const char *pszThread) -{ - std::string message = FormatException(pex, pszThread); - LogPrintf("\n\n************************\n%s\n", message); - fprintf(stderr, "\n\n************************\n%s\n", message.c_str()); - throw; -} - - -void PrintExceptionContinue(const std::exception *pex, const char *pszThread) -{ - std::string message = FormatException(pex, pszThread); - LogPrintf("\n\n************************\n%s\n", message); - fprintf(stderr, "\n\n************************\n%s\n", message.c_str()); -} - fs::path GetDefaultDataDir() { namespace fs = boost::filesystem; @@ -606,30 +410,6 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) #endif } -void ShrinkDebugFile() -{ - // Scroll debug.log if it's getting too big - fs::path pathLog = GetDataDir() / "debug.log"; - FILE *file = fopen(pathLog.string().c_str(), "r"); - if (file && fs::file_size(pathLog) > 20000) - { - // Restart the file with some of the end - std::vector vch(200000, 0); - fseek(file, -((long)vch.size()), SEEK_END); - int nBytes = fread(begin_ptr(vch), 1, vch.size(), file); - fclose(file); - - file = fopen(pathLog.string().c_str(), "w"); - if (file) - { - fwrite(begin_ptr(vch), 1, nBytes, file); - fclose(file); - } - } - else if (file != NULL) - fclose(file); -} - #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate) { @@ -714,6 +494,9 @@ void SetupEnvironment() // fs::path, which is then used to explicitly imbue the path. std::locale loc = fs::path::imbue(std::locale::classic()); fs::path::imbue(loc); + + // create the here and delete it absolutely last + g_logger = new CLogger(); } bool SetupNetworking() diff --git a/src/util/util.h b/src/util/util.h index 03f95089..056886c4 100644 --- a/src/util/util.h +++ b/src/util/util.h @@ -20,7 +20,7 @@ /** * Server/client environment: argument handling, config file parsing, - * logging, thread wrappers + * thread wrappers */ #ifndef BITCOIN_UTIL_H #define BITCOIN_UTIL_H @@ -47,94 +47,17 @@ // that is unique in this file. #define UNIQUIFY(pfx) UNIQUE1(pfx, __LINE__) -static const bool DEFAULT_LOGTIMEMICROS = false; -static const bool DEFAULT_LOGIPS = false; -static const bool DEFAULT_LOGTIMESTAMPS = true; - -/** Signals for translation. */ -class CTranslationInterface -{ -public: - /** Translate a message to the native language of the user. */ - boost::signals2::signal Translate; -}; - -extern bool fDebug; extern bool fDaemon; -extern bool fPrintToConsole; -extern bool fPrintToDebugLog; extern bool fServer; extern std::string strMiscWarning; -extern bool fLogTimestamps; -extern bool fLogTimeMicros; -extern bool fLogIPs; -extern volatile bool fReopenDebugLog; -extern CTranslationInterface translationInterface; extern const char *const CONF_FILENAME; extern const char *const PID_FILENAME; -/** - * Translation function: Call Translate signal on UI interface, which returns a boost::optional result. - * If no translation slot is registered, nothing is returned, and simply return the input. - */ -inline std::string _(const char *psz) -{ - boost::optional rv = translationInterface.Translate(psz); - return rv ? (*rv) : psz; -} void SetupEnvironment(); bool SetupNetworking(); -/** Return true if log accepts specified category */ -bool LogAcceptCategory(const char *category); -/** Send a string to the log output */ -int LogPrintStr(const std::string &str); - -#define LogPrintf(...) LogPrint(NULL, __VA_ARGS__) - -/** - * When we switch to C++11, this can be switched to variadic templates instead - * of this macro-based construction (see tinyformat.h). - */ -#define MAKE_ERROR_AND_LOG_FUNC(n) \ - /** Print to debug.log if -debug=category switch is given OR category is NULL. */ \ - template \ - static inline int LogPrint(const char *category, const char *format, TINYFORMAT_VARARGS(n)) \ - { \ - if (!LogAcceptCategory(category)) \ - return 0; \ - return LogPrintStr(tfm::format(format, TINYFORMAT_PASSARGS(n))); \ - } \ - /** Log error and return false */ \ - template \ - static inline bool error(const char *format, TINYFORMAT_VARARGS(n)) \ - { \ - LogPrintStr("ERROR: " + tfm::format(format, TINYFORMAT_PASSARGS(n)) + "\n"); \ - return false; \ - } - -TINYFORMAT_FOREACH_ARGNUM(MAKE_ERROR_AND_LOG_FUNC) - -/** - * Zero-arg versions of logging and error, these are not covered by - * TINYFORMAT_FOREACH_ARGNUM - */ -static inline int LogPrint(const char *category, const char *format) -{ - if (!LogAcceptCategory(category)) - return 0; - return LogPrintStr(format); -} -static inline bool error(const char *format) -{ - LogPrintStr(std::string("ERROR: ") + format + "\n"); - return false; -} - -void PrintException(const std::exception *pex, const char *pszThread); -void PrintExceptionContinue(const std::exception *pex, const char *pszThread); void ParseParameters(int argc, const char *const argv[]); void FileCommit(FILE *fileout); bool TruncateFile(FILE *file, unsigned int length); @@ -153,8 +76,6 @@ void CreatePidFile(const fs::path &path, pid_t pid); fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif fs::path GetTempPath(); -void OpenDebugLog(); -void ShrinkDebugFile(); void runCommand(const std::string &strCommand); inline bool IsSwitchChar(char c) @@ -194,37 +115,6 @@ int GetNumCores(); void SetThreadPriority(int nPriority); void RenameThread(const char *name); -/** - * .. and a wrapper that just calls func once - */ -template -void TraceThread(const char *name, Callable func) -{ - std::string s = strprintf("Eccoin-%s", name); - RenameThread(s.c_str()); - try - { - LogPrintf("%s thread start\n", name); - func(); - LogPrintf("%s thread exit\n", name); - } - catch (const boost::thread_interrupted &) - { - LogPrintf("%s thread interrupt\n", name); - throw; - } - catch (const std::exception &e) - { - PrintExceptionContinue(&e, name); - throw; - } - catch (...) - { - PrintExceptionContinue(NULL, name); - throw; - } -} - inline int64_t roundint64(double d) { return (int64_t)(d > 0 ? d + 0.5 : d - 0.5); } bool WildcardMatch(const char *psz, const char *mask); bool WildcardMatch(const std::string &str, const std::string &mask); diff --git a/src/verifydb.cpp b/src/verifydb.cpp index 098d8490..715650ed 100644 --- a/src/verifydb.cpp +++ b/src/verifydb.cpp @@ -25,8 +25,8 @@ #include "processblock.h" -CVerifyDB::CVerifyDB() { uiInterface.ShowProgress(_("Verifying blocks..."), 0); } -CVerifyDB::~CVerifyDB() { uiInterface.ShowProgress("", 100); } +CVerifyDB::CVerifyDB() {} +CVerifyDB::~CVerifyDB() {} bool CVerifyDB::VerifyDB(const CNetworkTemplate &chainparams, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) { LOCK(cs_main); @@ -54,10 +54,6 @@ bool CVerifyDB::VerifyDB(const CNetworkTemplate &chainparams, CCoinsView *coinsv LogPrintf("VerifyDB(): Shutdown requested. Exiting.\n"); return false; } - uiInterface.ShowProgress(_("Verifying blocks..."), - std::max(1, std::min( - 99, (int)(((double)(pnetMan->getChainActive()->chainActive.Height() - pindex->nHeight)) / - (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100))))); if (pindex->nHeight < pnetMan->getChainActive()->chainActive.Height() - nCheckDepth) break; CBlock block; @@ -120,10 +116,6 @@ bool CVerifyDB::VerifyDB(const CNetworkTemplate &chainparams, CCoinsView *coinsv LogPrintf("VerifyDB(): [lower] Shutdown requested. Exiting.\n"); return false; } - uiInterface.ShowProgress(_("Verifying blocks..."), - std::max(1, std::min(99, 100 - (int)(((double)(pnetMan->getChainActive()->chainActive.Height() - - pindex->nHeight)) / - (double)nCheckDepth * 50)))); pindex = pnetMan->getChainActive()->chainActive.Next(pindex); CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 50bd7724..c4cf6ad7 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -23,6 +23,7 @@ #include "crypto/aes.h" #include "script/script.h" #include "script/standard.h" +#include "util/logger.h" #include "util/util.h" #include diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 83868436..75846932 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -403,7 +403,7 @@ bool CWallet::Verify(const std::string &walletFile, std::string &warningString, if (!bitdb.Open(GetDataDir())) { // if it still fails, it probably means we can't even create the database env - std::string msg = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir()); + std::string msg = strprintf("Error initializing wallet database environment %s!", GetDataDir()); errorString += msg; return true; } @@ -421,14 +421,14 @@ bool CWallet::Verify(const std::string &walletFile, std::string &warningString, CDBEnv::VerifyResult r = bitdb.Verify(walletFile, CWalletDB::Recover); if (r == CDBEnv::RECOVER_OK) { - warningString += strprintf(_("Warning: wallet.dat corrupt, data salvaged!" - " Original wallet.dat saved as wallet.{timestamp}.bak in %s; if" - " your balance or transactions are incorrect you should" - " restore from a backup."), + warningString += strprintf(("Warning: wallet.dat corrupt, data salvaged!" + " Original wallet.dat saved as wallet.{timestamp}.bak in %s; if" + " your balance or transactions are incorrect you should" + " restore from a backup."), GetDataDir()); } if (r == CDBEnv::RECOVER_FAIL) - errorString += _("wallet.dat corrupt, salvage failed"); + errorString += ("wallet.dat corrupt, salvage failed"); } return true; @@ -767,9 +767,6 @@ bool CWallet::AddToWallet(const CWalletTx &wtxIn, bool fFromLoadWallet, CWalletD // Break debit/credit balance caches: wtx.MarkDirty(); - // Notify UI of new or updated transaction - NotifyTransactionChanged(this, hash, fInsertedNew ? CT_NEW : CT_UPDATED); - // notify an external script when a wallet transaction comes in or is updated std::string strCmd = gArgs.GetArg("-walletnotify", ""); @@ -872,7 +869,6 @@ bool CWallet::AbandonTransaction(const uint256 &hashTx) wtx.setAbandoned(); wtx.MarkDirty(); wtx.WriteToDisk(&walletdb); - NotifyTransactionChanged(this, wtx.tx->GetHash(), CT_UPDATED); // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too TxSpends::const_iterator iter = mapTxSpends.lower_bound(COutPoint(hashTx, 0)); while (iter != mapTxSpends.end() && iter->first.hash == now) @@ -1256,7 +1252,7 @@ int CWallet::ScanForWalletTransactions(CBlockIndex *pindexStart, bool fUpdate) } // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup - ShowProgress(_("Rescanning..."), 0); + ShowProgress(("Rescanning..."), 0); double dProgressStart = Checkpoints::GuessVerificationProgress(pnetMan->getActivePaymentNetwork()->Checkpoints(), pindex, false); double dProgressTip = Checkpoints::GuessVerificationProgress( @@ -1265,7 +1261,7 @@ int CWallet::ScanForWalletTransactions(CBlockIndex *pindexStart, bool fUpdate) { if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) { - ShowProgress(_("Rescanning..."), + ShowProgress(("Rescanning..."), std::max(1, std::min( 99, (int)((Checkpoints::GuessVerificationProgress( pnetMan->getActivePaymentNetwork()->Checkpoints(), pindex, false) - @@ -1290,7 +1286,7 @@ int CWallet::ScanForWalletTransactions(CBlockIndex *pindexStart, bool fUpdate) Checkpoints::GuessVerificationProgress(pnetMan->getActivePaymentNetwork()->Checkpoints(), pindex)); } } - ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI + ShowProgress(("Rescanning..."), 100); // hide progress dialog in GUI } return ret; } @@ -2119,7 +2115,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, { if (nValue < 0 || recipient.nAmount < 0) { - strFailReason = _("Transaction amounts must be positive"); + strFailReason = ("Transaction amounts must be positive"); return false; } nValue += recipient.nAmount; @@ -2129,7 +2125,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, } if (vecSend.empty() || nValue < 0) { - strFailReason = _("Transaction amounts must be positive"); + strFailReason = ("Transaction amounts must be positive"); return false; } @@ -2208,13 +2204,13 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { if (txout.nValue < 0) - strFailReason = _("The transaction amount is too small to pay the fee"); + strFailReason = ("The transaction amount is too small to pay the fee"); else strFailReason = - _("The transaction amount is too small to send after the fee has been deducted"); + ("The transaction amount is too small to send after the fee has been deducted"); } else - strFailReason = _("Transaction amount too small"); + strFailReason = ("Transaction amount too small"); return false; } txNew.vout.push_back(txout); @@ -2225,7 +2221,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, CAmount nValueIn = 0; if (!SelectCoins(nValueToSelect, setCoins, nValueIn)) { - strFailReason = _("Insufficient funds"); + strFailReason = ("Insufficient funds"); return false; } for (auto pcoin : setCoins) @@ -2317,8 +2313,8 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, txNew.vout[i].nValue -= nDust; if (txNew.vout[i].IsDust(::minRelayTxFee)) { - strFailReason = _( - "The transaction amount is too small to send after the fee has been deducted"); + strFailReason = + "The transaction amount is too small to send after the fee has been deducted"; return false; } break; @@ -2370,7 +2366,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, if (!signSuccess) { - strFailReason = _("Signing transaction failed"); + strFailReason = ("Signing transaction failed"); return false; } nIn++; @@ -2393,7 +2389,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, // Limit size if (nBytes >= MAX_STANDARD_TX_SIZE) { - strFailReason = _("Transaction too large"); + strFailReason = ("Transaction too large"); return false; } @@ -2415,7 +2411,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, // because we must be at the maximum allowed fee. if (nFeeNeeded < ::minRelayTxFee.GetFee(nBytes)) { - strFailReason = _("Transaction too large for fee policy"); + strFailReason = ("Transaction too large for fee policy"); return false; } @@ -2470,7 +2466,6 @@ bool CWallet::CommitTransaction(CWalletTx &wtxNew, CReserveKey &reservekey, CCon { CWalletTx &coin = mapWallet[txin.prevout.hash]; coin.BindWallet(this); - NotifyTransactionChanged(this, coin.tx->GetHash(), CT_UPDATED); } if (fFileBacked) @@ -2538,8 +2533,6 @@ DBErrors CWallet::LoadWallet(bool &fFirstRunRet) return nLoadWalletRet; fFirstRunRet = !vchDefaultKey.IsValid(); - uiInterface.LoadWallet(this); - return DB_LOAD_OK; } @@ -2579,8 +2572,6 @@ bool CWallet::SetAddressBook(const CTxDestination &address, const std::string &s if (!strPurpose.empty()) /* update purpose only if requested */ mapAddressBook[address].purpose = strPurpose; } - NotifyAddressBookChanged( - this, address, strName, ::IsMine(*this, address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW)); if (!fFileBacked) return false; if (!strPurpose.empty() && !CWalletDB(strWalletFile).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) @@ -2611,8 +2602,6 @@ bool CWallet::DelAddressBook(const CTxDestination &address) mapAddressBook.erase(address); } - NotifyAddressBookChanged(this, address, "", ::IsMine(*this, address) != ISMINE_NO, "", CT_DELETED); - if (!fFileBacked) return false; CWalletDB(strWalletFile).ErasePurpose(CBitcoinAddress(address).ToString()); @@ -2712,10 +2701,7 @@ void CWallet::ReserveKeyFromKeyPool(int64_t &nIndex, CKeyPool &keypool) if (!HaveKey(keypool.vchPubKey.GetID())) throw std::runtime_error("ReserveKeyFromKeyPool(): unknown key in key pool"); assert(keypool.vchPubKey.IsValid()); - if (fDebug) - { - LogPrintf("keypool reserve %d\n", nIndex); - } + LogPrint("wallet", "keypool reserve %d\n", nIndex); } } @@ -2737,10 +2723,7 @@ void CWallet::ReturnKey(int64_t nIndex) LOCK(cs_wallet); setKeyPool.insert(nIndex); } - if (fDebug) - { - LogPrintf("keypool return %d\n", nIndex); - } + LogPrint("wallet", "keypool return %d\n", nIndex); } bool CWallet::GetKeyFromPool(CPubKey &result) @@ -2975,17 +2958,6 @@ void CWallet::GetAllReserveKeys(std::set &setAddress) const } } -void CWallet::UpdatedTransaction(const uint256 &hashTx) -{ - { - LOCK(cs_wallet); - // Only notify UI if this transaction is in this wallet - std::map::const_iterator mi = mapWallet.find(hashTx); - if (mi != mapWallet.end()) - NotifyTransactionChanged(this, hashTx, CT_UPDATED); - } -} - void CWallet::GetScriptForMining(boost::shared_ptr &script) { boost::shared_ptr rKey(new CReserveKey(this)); @@ -3353,7 +3325,7 @@ bool CWallet::SelectCoinsMinConf(CAmount nTargetValue, nValueRet += vValue[i].first; } - if (fDebug && gArgs.GetBoolArg("-printpriority", false)) + if (gArgs.GetBoolArg("-printpriority", false)) { //// debug print LogPrintf("SelectCoins() best subset: "); @@ -3461,24 +3433,20 @@ bool CWallet::CreateCoinStake(const CKeyStore &keystore, txindex.nTxOffset, *(pcoin.first->tx), prevoutStake, txNew.nTime, hashProofOfStake)) { // Found a kernel - if (fDebug) - LogPrintf("CreateCoinStake : kernel found\n"); + LogPrint("wallet", "CreateCoinStake : kernel found\n"); std::vector > vSolutions; txnouttype whichType; CScript scriptPubKeyOut; scriptPubKeyKernel = pcoin.first->tx->vout[pcoin.second].scriptPubKey; if (!Solver(scriptPubKeyKernel, whichType, vSolutions)) { - if (fDebug) - LogPrintf("CreateCoinStake : failed to parse kernel\n"); + LogPrint("wallet", "CreateCoinStake : failed to parse kernel\n"); break; } - if (fDebug) - LogPrintf("CreateCoinStake : parsed kernel type=%d\n", whichType); + LogPrint("wallet", "CreateCoinStake : parsed kernel type=%d\n", whichType); if (whichType != TX_PUBKEY && whichType != TX_PUBKEYHASH) { - if (fDebug) - LogPrintf("CreateCoinStake : no support for kernel type=%d\n", whichType); + LogPrint("wallet", "CreateCoinStake : no support for kernel type=%d\n", whichType); break; // only support pay to public key and pay to address } if (whichType == TX_PUBKEYHASH) // pay to address type @@ -3487,8 +3455,7 @@ bool CWallet::CreateCoinStake(const CKeyStore &keystore, CKey key; if (!keystore.GetKey(uint160(vSolutions[0]), key)) { - if (fDebug) - LogPrintf("CreateCoinStake : failed to get key for kernel type=%d\n", whichType); + LogPrint("wallet", "CreateCoinStake : failed to get key for kernel type=%d\n", whichType); break; // unable to find corresponding public key } scriptPubKeyOut << key.GetPubKey() << OP_CHECKSIG; @@ -3503,8 +3470,7 @@ bool CWallet::CreateCoinStake(const CKeyStore &keystore, if (block.GetBlockTime() + nStakeSplitAge > txNew.nTime) txNew.vout.push_back(CTxOut(0, scriptPubKeyOut)); // split stake - if (fDebug) - LogPrintf("CreateCoinStake : added kernel type=%d\n", whichType); + LogPrint("wallet", "CreateCoinStake : added kernel type=%d\n", whichType); fKernelFound = true; break; } @@ -3605,15 +3571,11 @@ bool CWallet::CreateCoinStake(const CKeyStore &keystore, bool static UIError(const std::string &str) { - uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_ERROR); + LogPrintf("Wallet Error: %s\n", str.c_str()); return false; } -void static UIWarning(const std::string &str) -{ - uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_WARNING); -} - +void static UIWarning(const std::string &str) { LogPrintf("Wallet Warning: %s\n", str.c_str()); } bool CWallet::InitLoadWallet() { std::string walletFile = gArgs.GetArg("-wallet", DEFAULT_WALLET_DAT); @@ -3629,7 +3591,7 @@ bool CWallet::InitLoadWallet() DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DB_LOAD_OK) { - return UIError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + return UIError(strprintf(("Error loading %s: Wallet corrupted"), walletFile)); } delete tempWallet; @@ -3646,26 +3608,26 @@ bool CWallet::InitLoadWallet() { if (nLoadWalletRet == DB_CORRUPT) { - return UIError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + return UIError(strprintf(("Error loading %s: Wallet corrupted"), walletFile)); } else if (nLoadWalletRet == DB_NONCRITICAL_ERROR) { - UIWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" - " or address book entries might be missing or incorrect."), + UIWarning(strprintf(("Error reading %s! All keys read correctly, but transaction data" + " or address book entries might be missing or incorrect."), walletFile)); } else if (nLoadWalletRet == DB_TOO_NEW) { return UIError( - strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, _(PACKAGE_NAME))); + strprintf(("Error loading %s: Wallet requires newer version of %s"), walletFile, PACKAGE_NAME)); } else if (nLoadWalletRet == DB_NEED_REWRITE) { - return UIError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); + return UIError(strprintf(("Wallet needed to be rewritten: restart %s to complete"), PACKAGE_NAME)); } else { - return UIError(strprintf(_("Error loading %s"), walletFile)); + return UIError(strprintf(("Error loading %s"), walletFile)); } } @@ -3684,7 +3646,7 @@ bool CWallet::InitLoadWallet() } if (nMaxVersion < walletInstance->GetVersion()) { - return UIError(_("Cannot downgrade wallet")); + return UIError(("Cannot downgrade wallet")); } walletInstance->SetMaxVersion(nMaxVersion); } @@ -3699,7 +3661,7 @@ bool CWallet::InitLoadWallet() { walletInstance->SetDefaultKey(newDefaultKey); if (!walletInstance->SetAddressBook(walletInstance->vchDefaultKey.GetID(), "", "receive")) - return UIError(_("Cannot write default address") += "\n"); + return UIError("Cannot write default address \n"); } walletInstance->SetBestChain(pnetMan->getChainActive()->chainActive.GetLocator()); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index dd4938f2..1942b981 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -27,7 +27,7 @@ #include "policy/policy.h" #include "streams.h" #include "tinyformat.h" -#include "ui_interface.h" + #include "util/utilstrencodings.h" #include "validationinterface.h" #include "wallet/crypter.h" @@ -775,8 +775,6 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface bool DelAddressBook(const CTxDestination &address); - void UpdatedTransaction(const uint256 &hashTx); - void Inventory(const uint256 &hash) { { @@ -826,24 +824,6 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface //! Verify the wallet database and perform salvage if required static bool Verify(const std::string &walletFile, std::string &warningString, std::string &errorString); - /** - * Address book entry changed. - * @note called with lock cs_wallet held. - */ - boost::signals2::signal - NotifyAddressBookChanged; - - /** - * Wallet transaction added, removed or updated. - * @note called with lock cs_wallet held. - */ - boost::signals2::signal NotifyTransactionChanged; - /** Show progress e.g. for rescan */ boost::signals2::signal ShowProgress; From 8af93ba642b5b26c8103069256ed61eb27a189e2 Mon Sep 17 00:00:00 2001 From: Henry Young <8385576+HenryYoung42@users.noreply.github.com> Date: Mon, 18 Feb 2019 20:44:45 +0000 Subject: [PATCH 08/46] Removal of unused variables (#126) * Removal of unused variable amount * Removal of unused variables in rpcmining --- src/rpc/rpcmining.cpp | 6 ------ src/rpc/rpcrawtransaction.cpp | 1 - 2 files changed, 7 deletions(-) diff --git a/src/rpc/rpcmining.cpp b/src/rpc/rpcmining.cpp index e439b660..711aa4f1 100644 --- a/src/rpc/rpcmining.cpp +++ b/src/rpc/rpcmining.cpp @@ -335,9 +335,6 @@ UniValue generatetoaddress(const UniValue ¶ms, bool fHelp) throw JSONRPCError(RPC_METHOD_NOT_FOUND, "This method can only be used on regtest"); } - int nHeightStart = 0; - int nHeightEnd = 0; - int nHeight = 0; int nGenerate = params[0].get_int(); uint64_t nMaxTries = 1000000; @@ -386,9 +383,6 @@ UniValue generatepostoaddress(const UniValue ¶ms, bool fHelp) throw JSONRPCError(RPC_METHOD_NOT_FOUND, "This method can only be used on regtest"); } - int nHeightStart = 0; - int nHeightEnd = 0; - int nHeight = 0; int nGenerate = params[0].get_int(); uint64_t nMaxTries = 1000000; diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 7e453403..04e7085e 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -808,7 +808,6 @@ UniValue signrawtransaction(const UniValue ¶ms, bool fHelp) continue; } const CScript &prevPubKey = coin.out.scriptPubKey; - const CAmount &amount = coin.out.nValue; txin.scriptSig.clear(); // Only sign SIGHASH_SINGLE if there's a corresponding output: From b1978a10b0f297148f898d611c186a83ae31cd05 Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Sun, 24 Feb 2019 02:07:28 -0500 Subject: [PATCH 09/46] reimplement zmq (#131) --- .travis.yml | 3 +- doc/zmq.md | 101 +++++++++++++++ qa/pull-tester/rpc-tests.py | 4 +- src/.formatted-files | 7 ++ src/Makefile.am | 26 +++- src/Makefile.bench.include | 4 + src/Makefile.test.include | 8 +- src/args.cpp | 6 + src/args.h | 2 + src/init.cpp | 26 +++- src/net/messages.cpp | 40 ------ src/net/messages.h | 2 - src/processblock.cpp | 2 +- src/validationinterface.cpp | 4 +- src/validationinterface.h | 4 +- src/zmq/zmqabstractnotifier.cpp | 12 ++ src/zmq/zmqabstractnotifier.h | 44 +++++++ src/zmq/zmqconfig.h | 25 ++++ src/zmq/zmqnotificationinterface.cpp | 155 +++++++++++++++++++++++ src/zmq/zmqnotificationinterface.h | 38 ++++++ src/zmq/zmqpublishnotifier.cpp | 177 +++++++++++++++++++++++++++ src/zmq/zmqpublishnotifier.h | 44 +++++++ 22 files changed, 677 insertions(+), 57 deletions(-) create mode 100644 doc/zmq.md create mode 100644 src/zmq/zmqabstractnotifier.cpp create mode 100644 src/zmq/zmqabstractnotifier.h create mode 100644 src/zmq/zmqconfig.h create mode 100644 src/zmq/zmqnotificationinterface.cpp create mode 100644 src/zmq/zmqnotificationinterface.h create mode 100644 src/zmq/zmqpublishnotifier.cpp create mode 100644 src/zmq/zmqpublishnotifier.h diff --git a/.travis.yml b/.travis.yml index 9dd09df6..8e0038a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,7 +78,7 @@ matrix: - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=x86_64-unknown-linux-gnu PACKAGES="python3 qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev" DEP_OPTS="NO_WALLET=1 NO_QT=1 ALLOW_HOST_PACKAGES=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=x86_64-unknown-linux-gnu PACKAGES="python3 python3-zmq libzmq3-dev qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev" DEP_OPTS="NO_WALLET=1 NO_QT=1 ALLOW_HOST_PACKAGES=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" #Cross-Mac - compiler: gcc env: @@ -142,4 +142,3 @@ after_failure: - for i in `find /tmp/ -name debug.log`; do echo $i; echo "-----"; cat $i; done - for i in `find /home/travis/ -name bitcoin.conf`; do echo $i; echo "-----"; cat $i; done - for i in `find /tmp/ -name bitcoin.conf`; do echo $i; echo "-----"; cat $i; done - diff --git a/doc/zmq.md b/doc/zmq.md new file mode 100644 index 00000000..af8d9e83 --- /dev/null +++ b/doc/zmq.md @@ -0,0 +1,101 @@ +# Block and Transaction Broadcasting With ZeroMQ + +[ZeroMQ](http://zeromq.org/) is a lightweight wrapper around TCP +connections, inter-process communication, and shared-memory, +providing various message-oriented semantics such as publish/subscribe, +request/reply, and push/pull. + +The Bitcoin Unlimited daemon can be configured to act as a trusted "border +router", implementing the bitcoin wire protocol and relay, making +consensus decisions, maintaining the local blockchain database, +broadcasting locally generated transactions into the network, and +providing a queryable RPC interface to interact on a polled basis for +requesting blockchain related data. However, there exists only a +limited service to notify external software of events like the arrival +of new blocks or transactions. + +The ZeroMQ facility implements a notification interface through a set +of specific notifiers. Currently there are notifiers that publish +blocks and transactions. This read-only facility requires only the +connection of a corresponding ZeroMQ subscriber port in receiving +software; it is not authenticated nor is there any two-way protocol +involvement. Therefore, subscribers should validate the received data +since it may be out of date, incomplete or even invalid. + +ZeroMQ sockets are self-connecting and self-healing; that is, +connections made between two endpoints will be automatically restored +after an outage, and either end may be freely started or stopped in +any order. + +Because ZeroMQ is message oriented, subscribers receive transactions +and blocks all-at-once and do not need to implement any sort of +buffering or reassembly. + +## Prerequisites + +The ZeroMQ feature in Bitcoin Unlimited requires ZeroMQ API version 4.x or +newer. Typically, it is packaged by distributions as something like +*libzmq3-dev*. The C++ wrapper for ZeroMQ is *not* needed. + +In order to run the example Python client scripts in contrib/ one must +also install *python3-zmq*, though this is not necessary for daemon +operation. + +## Enabling + +By default, the ZeroMQ feature is automatically compiled in if the +necessary prerequisites are found. To disable, use --disable-zmq +during the *configure* step of building bitcoind: + + $ ./configure --disable-zmq (other options) + +To actually enable operation, one must set the appropriate options on +the commandline or in the configuration file. + +## Usage + +Currently, the following notifications are supported: + + -zmqpubhashtx=address + -zmqpubhashblock=address + -zmqpubrawblock=address + -zmqpubrawtx=address + +The socket type is PUB and the address must be a valid ZeroMQ socket +address. The same address can be used in more than one notification. + +For instance: + + $ bitcoind -zmqpubhashtx=tcp://127.0.0.1:28332 \ + -zmqpubrawtx=ipc:///tmp/bitcoind.tx.raw + +Each PUB notification has a topic and body, where the header +corresponds to the notification type. For instance, for the +notification `-zmqpubhashtx` the topic is `hashtx` (no null +terminator) and the body is the hexadecimal transaction hash (32 +bytes). + +These options can also be provided in bitcoin.conf. + +ZeroMQ endpoint specifiers for TCP (and others) are documented in the +[ZeroMQ API](http://api.zeromq.org/4-0:_start). + +Client side, then, the ZeroMQ subscriber socket must have the +ZMQ_SUBSCRIBE option set to one or either of these prefixes (for +instance, just `hash`); without doing so will result in no messages +arriving. Please see `contrib/zmq/zmq_sub.py` for a working example. + +## Remarks + +From the perspective of bitcoind, the ZeroMQ socket is write-only; PUB +sockets don't even have a read function. Thus, there is no state +introduced into bitcoind directly. Furthermore, no information is +broadcast that wasn't already received from the public P2P network. + +No authentication or authorization is done on connecting clients; it +is assumed that the ZeroMQ port is exposed only to trusted entities, +using other means such as firewalling. + +Note that when the block chain tip changes, a reorganisation may occur +and just the tip will be notified. It is up to the subscriber to +retrieve the chain from the last known block to the new tip. diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 58fb547f..7db39b2d 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -61,8 +61,8 @@ ENABLE_BITCOIND=0 if 'ENABLE_UTILS' not in vars(): ENABLE_UTILS=0 -#if 'ENABLE_ZMQ' not in vars(): -ENABLE_ZMQ=0 +if 'ENABLE_ZMQ' not in vars(): + ENABLE_ZMQ=0 ENABLE_COVERAGE=0 diff --git a/src/.formatted-files b/src/.formatted-files index edc31053..92352a1c 100644 --- a/src/.formatted-files +++ b/src/.formatted-files @@ -197,6 +197,13 @@ wallet/wallet.cpp wallet/wallet.h wallet/walletdb.cpp wallet/walletdb.h +zmq/zmqabstractnotifier.cpp +zmq/zmqabstractnotifier.h +zmq/zmqconfig.h +zmq/zmqnotificationinterface.cpp +zmq/zmqnotificationinterface.h +zmq/zmqpublishnotifier.cpp +zmq/zmqpublishnotifier.h test/addrman_tests.cpp test/allocator_tests.cpp test/arith_uint256_tests.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 37b2db48..0cec29a2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -15,6 +15,10 @@ LIBBITCOIN_SERVER=libbitcoin_server.a LIBSECP256K1=secp256k1/libsecp256k1.la LIBUNIVALUE=univalue/libunivalue.la +if ENABLE_ZMQ +LIBBITCOIN_ZMQ=libbitcoin_zmq.a +endif + $(LIBSECP256K1): $(wildcard secp256k1/src/*) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) @@ -25,7 +29,8 @@ $(LIBUNIVALUE): $(wildcard univalue/lib/*) $(wildcard univalue/include/*) # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES += \ - $(LIBBITCOIN_SERVER) + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_ZMQ) bin_PROGRAMS = TESTS = @@ -146,7 +151,11 @@ BITCOIN_CORE_H = \ wallet/db.h \ wallet/wallet.h \ wallet/wallet_ismine.h \ - wallet/walletdb.h + wallet/walletdb.h \ + zmq/zmqabstractnotifier.h \ + zmq/zmqconfig.h\ + zmq/zmqnotificationinterface.h \ + zmq/zmqpublishnotifier.h build/build.h: FORCE @$(MKDIR_P) $(builddir)/build @@ -271,6 +280,15 @@ libbitcoin_server_a_SOURCES = \ rpc/rpcclient.cpp \ $(BITCOIN_CORE_H) +if ENABLE_ZMQ +libbitcoin_zmq_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) +libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +libbitcoin_zmq_a_SOURCES = \ + zmq/zmqabstractnotifier.cpp \ + zmq/zmqnotificationinterface.cpp \ + zmq/zmqpublishnotifier.cpp +endif + if GLIBC_BACK_COMPAT libbitcoin_server_a_SOURCES += compat/glibc_compat.cpp endif @@ -283,6 +301,7 @@ eccoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) eccoind_LDADD = \ $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CRYPTO) \ $(LIBUNIVALUE) \ $(LIBLEVELDB) \ @@ -290,7 +309,7 @@ eccoind_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) -eccoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) +eccoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) # CLEANFILES = $(EXTRA_LIBRARIES) @@ -312,6 +331,7 @@ CLEANFILES += support/*.gcda support/*.gcno CLEANFILES += univalue/*.gcda univalue/*.gcno CLEANFILES += wallet/*.gcda wallet/*.gcno CLEANFILES += wallet/test/*.gcda wallet/test/*.gcno +CLEANFILES += zmq/*.gcda zmq/*.gcno DISTCLEANFILES = build/build.h diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 68698180..73653e9d 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -23,6 +23,10 @@ bench_bench_bitcoin_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) +if ENABLE_ZMQ +bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +endif + bench_bench_bitcoin_LDADD += $(LIBBITCOIN_WALLET) bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 5ed73237..6fbd9144 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -113,6 +113,7 @@ test_test_bitcoin_LDADD = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBUNIVALUE) \ + $(LIBBITCOIN_ZMQ) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ @@ -131,6 +132,10 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) test_test_bitcoin_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static +if ENABLE_ZMQ +test_test_bitcoin_LDADD += $(ZMQ_LIBS) +endif + # # test_bitcoin_fuzzy binary # @@ -143,6 +148,7 @@ test_test_bitcoin_fuzzy_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ $(LIBUNIVALUE) \ + $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_CONSENSUS) \ @@ -152,7 +158,7 @@ test_test_bitcoin_fuzzy_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) -test_test_bitcoin_fuzzy_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) +test_test_bitcoin_fuzzy_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) # nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/args.cpp b/src/args.cpp index 80c888e1..6ce2a967 100644 --- a/src/args.cpp +++ b/src/args.cpp @@ -158,6 +158,12 @@ std::vector CArgsManager::GetArgs(const std::string &strArg) return {}; } +std::map CArgsManager::GetMapArgs() +{ + LOCK(cs_args); + return mapArgs; +} + bool CArgsManager::IsArgSet(const std::string &strArg) { LOCK(cs_args); diff --git a/src/args.h b/src/args.h index 92dbdc49..47ff03e5 100644 --- a/src/args.h +++ b/src/args.h @@ -47,6 +47,8 @@ class CArgsManager void ReadConfigFile(); std::vector GetArgs(const std::string &strArg); + std::map GetMapArgs(); + /** * Return true if the given argument has been manually set * diff --git a/src/init.cpp b/src/init.cpp index 159ff0f9..76b1d157 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -72,6 +72,11 @@ #include +#if ENABLE_ZMQ +#include "zmq/zmqnotificationinterface.h" +static CZMQNotificationInterface *pzmqNotificationInterface = nullptr; +#endif + CWallet *pwalletMain = nullptr; CNetworkManager *pnetMan = nullptr; @@ -262,6 +267,15 @@ void Shutdown() pwalletMain->Flush(true); } +#if ENABLE_ZMQ + if (pzmqNotificationInterface) + { + UnregisterValidationInterface(pzmqNotificationInterface); + delete pzmqNotificationInterface; + pzmqNotificationInterface = NULL; + } +#endif + #ifndef WIN32 try { @@ -1594,15 +1608,23 @@ bool AppInit2(thread_group &threadGroup) } threadGroup.create_thread("importFiles", &ThreadImport, vImportFiles); - if (pnetMan->getChainActive()->chainActive.Tip() == NULL) + if (pnetMan->getChainActive()->chainActive.Tip() == nullptr) { LogPrintf("Waiting for genesis block to be imported...\n"); - while (!shutdown_threads.load() && pnetMan->getChainActive()->chainActive.Tip() == NULL) + while (!shutdown_threads.load() && pnetMan->getChainActive()->chainActive.Tip() == nullptr) { MilliSleep(10); } } +#if ENABLE_ZMQ + pzmqNotificationInterface = CZMQNotificationInterface::CreateWithArguments(gArgs.GetMapArgs()); + if (pzmqNotificationInterface) + { + RegisterValidationInterface(pzmqNotificationInterface); + } +#endif + // ********************************************************* Step 11: start node if (!CheckDiskSpace()) diff --git a/src/net/messages.cpp b/src/net/messages.cpp index e20c1d9c..5096c7ec 100644 --- a/src/net/messages.cpp +++ b/src/net/messages.cpp @@ -739,46 +739,6 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const CBlo }); } -void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, - const CBlockIndex *pindexFork, - bool fInitialDownload) -{ - const int nNewHeight = pindexNew->nHeight; - connman->SetBestHeight(nNewHeight); - - if (!fInitialDownload) - { - // Find the hashes of all blocks that weren't previously in the best - // chain. - std::vector vHashes; - const CBlockIndex *pindexToAnnounce = pindexNew; - while (pindexToAnnounce != pindexFork) - { - vHashes.push_back(pindexToAnnounce->GetBlockHash()); - pindexToAnnounce = pindexToAnnounce->pprev; - if (vHashes.size() == MAX_BLOCKS_TO_ANNOUNCE) - { - // Limit announcements in case of a huge reorganization. Rely on - // the peer's synchronization mechanism in that case. - break; - } - } - // Relay inventory, but don't relay old inventory during initial block - // download. - connman->ForEachNode([nNewHeight, &vHashes](CNode *pnode) { - if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) - { - for (const uint256 &hash : boost::adaptors::reverse(vHashes)) - { - pnode->PushBlockHash(hash); - } - } - }); - } - - nTimeBestReceived = GetTime(); -} - bool AlreadyHave(const CInv &inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { switch (inv.type) diff --git a/src/net/messages.h b/src/net/messages.h index 0bfd5aed..8ec15bf2 100644 --- a/src/net/messages.h +++ b/src/net/messages.h @@ -62,8 +62,6 @@ class PeerLogicValidation : public CValidationInterface public: PeerLogicValidation(CConnman *connmanIn); - - void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; void NewPoWValidBlock(const CBlockIndex *pindex, const CBlock *pblock) override; }; diff --git a/src/processblock.cpp b/src/processblock.cpp index 179d50b8..562bcf39 100644 --- a/src/processblock.cpp +++ b/src/processblock.cpp @@ -673,7 +673,7 @@ bool ActivateBestChain(CValidationState &state, const CNetworkTemplate &chainpar // Notify external listeners about the new tip. if (!vHashes.empty()) { - GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); + GetMainSignals().UpdatedBlockTip(pindexNewTip); } } } diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index e4f8da9d..80a5d715 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -25,7 +25,7 @@ static CMainSignals g_signals; CMainSignals &GetMainSignals() { return g_signals; } void RegisterValidationInterface(CValidationInterface *pwalletIn) { - g_signals.UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); + g_signals.UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1)); g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); @@ -43,7 +43,7 @@ void UnregisterValidationInterface(CValidationInterface *pwalletIn) g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); - g_signals.UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); + g_signals.UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1)); g_signals.NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } diff --git a/src/validationinterface.h b/src/validationinterface.h index 1377f1bf..df30fad9 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -51,7 +51,7 @@ void SyncWithWallets(const CTransactionRef &ptx, const CBlock *pblock, int txIdx class CValidationInterface { protected: - virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} + virtual void UpdatedBlockTip(const CBlockIndex *pindex) {} virtual void SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock, int txIdx) {} virtual void SetBestChain(const CBlockLocator &locator) {} virtual void Inventory(const uint256 &hash) {} @@ -67,7 +67,7 @@ class CValidationInterface struct CMainSignals { /** Notifies listeners of updated block chain tip */ - boost::signals2::signal UpdatedBlockTip; + boost::signals2::signal UpdatedBlockTip; /** Notifies listeners of updated transaction data (transaction, and optionally the block it is found in. */ boost::signals2::signal SyncTransaction; /** Notifies listeners of a new active block chain. */ diff --git a/src/zmq/zmqabstractnotifier.cpp b/src/zmq/zmqabstractnotifier.cpp new file mode 100644 index 00000000..bc11e1aa --- /dev/null +++ b/src/zmq/zmqabstractnotifier.cpp @@ -0,0 +1,12 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "zmqabstractnotifier.h" +#include "util/util.h" + + +CZMQAbstractNotifier::~CZMQAbstractNotifier() { assert(!psocket); } +bool CZMQAbstractNotifier::NotifyBlock(const CBlockIndex * /*CBlockIndex*/) { return true; } +bool CZMQAbstractNotifier::NotifyTransaction(const CTransactionRef & /*transaction*/) { return true; } diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h new file mode 100644 index 00000000..79a6a2ab --- /dev/null +++ b/src/zmq/zmqabstractnotifier.h @@ -0,0 +1,44 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H +#define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H + +#include "zmqconfig.h" + +class CBlockIndex; +class CZMQAbstractNotifier; + +typedef CZMQAbstractNotifier *(*CZMQNotifierFactory)(); + +class CZMQAbstractNotifier +{ +public: + CZMQAbstractNotifier() : psocket(0) {} + virtual ~CZMQAbstractNotifier(); + + template + static CZMQAbstractNotifier *Create() + { + return new T(); + } + + std::string GetType() const { return type; } + void SetType(const std::string &t) { type = t; } + std::string GetAddress() const { return address; } + void SetAddress(const std::string &a) { address = a; } + virtual bool Initialize(void *pcontext) = 0; + virtual void Shutdown() = 0; + + virtual bool NotifyBlock(const CBlockIndex *pindex); + virtual bool NotifyTransaction(const CTransactionRef &ptx); + +protected: + void *psocket; + std::string type; + std::string address; +}; + +#endif // BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H diff --git a/src/zmq/zmqconfig.h b/src/zmq/zmqconfig.h new file mode 100644 index 00000000..af54f543 --- /dev/null +++ b/src/zmq/zmqconfig.h @@ -0,0 +1,25 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQCONFIG_H +#define BITCOIN_ZMQ_ZMQCONFIG_H + +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif + +#include +#include + +#if ENABLE_ZMQ +#include +#endif + +#include "chain/block.h" +#include "chain/tx.h" + +void zmqError(const char *str); + +#endif // BITCOIN_ZMQ_ZMQCONFIG_H diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp new file mode 100644 index 00000000..c821d997 --- /dev/null +++ b/src/zmq/zmqnotificationinterface.cpp @@ -0,0 +1,155 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "zmqnotificationinterface.h" +#include "zmqpublishnotifier.h" + +#include "main.h" +#include "streams.h" +#include "util/util.h" +#include "version.h" + +void zmqError(const char *str) { LogPrint("zmq", "zmq: Error: %s, errno=%s\n", str, zmq_strerror(errno)); } +CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(NULL) {} +CZMQNotificationInterface::~CZMQNotificationInterface() +{ + Shutdown(); + + for (std::list::iterator i = notifiers.begin(); i != notifiers.end(); ++i) + { + delete *i; + } +} + +CZMQNotificationInterface *CZMQNotificationInterface::CreateWithArguments( + const std::map &args) +{ + CZMQNotificationInterface *notificationInterface = NULL; + std::map factories; + std::list notifiers; + + factories["pubhashblock"] = CZMQAbstractNotifier::Create; + factories["pubhashtx"] = CZMQAbstractNotifier::Create; + factories["pubrawblock"] = CZMQAbstractNotifier::Create; + factories["pubrawtx"] = CZMQAbstractNotifier::Create; + + for (std::map::const_iterator i = factories.begin(); i != factories.end(); ++i) + { + std::map::const_iterator j = args.find("-zmq" + i->first); + if (j != args.end()) + { + CZMQNotifierFactory factory = i->second; + std::string address = j->second; + CZMQAbstractNotifier *notifier = factory(); + notifier->SetType(i->first); + notifier->SetAddress(address); + notifiers.push_back(notifier); + } + } + + if (!notifiers.empty()) + { + notificationInterface = new CZMQNotificationInterface(); + notificationInterface->notifiers = notifiers; + + if (!notificationInterface->Initialize()) + { + delete notificationInterface; + notificationInterface = NULL; + } + } + + return notificationInterface; +} + +// Called at startup to conditionally set up ZMQ socket(s) +bool CZMQNotificationInterface::Initialize() +{ + LogPrint("zmq", "zmq: Initialize notification interface\n"); + assert(!pcontext); + + pcontext = zmq_init(1); + + if (!pcontext) + { + zmqError("Unable to initialize context"); + return false; + } + + std::list::iterator i = notifiers.begin(); + for (; i != notifiers.end(); ++i) + { + CZMQAbstractNotifier *notifier = *i; + if (notifier->Initialize(pcontext)) + { + LogPrint("zmq", " Notifier %s ready (address = %s)\n", notifier->GetType(), notifier->GetAddress()); + } + else + { + LogPrint("zmq", " Notifier %s failed (address = %s)\n", notifier->GetType(), notifier->GetAddress()); + break; + } + } + + if (i != notifiers.end()) + { + Shutdown(); + return false; + } + + return true; +} + +// Called during shutdown sequence +void CZMQNotificationInterface::Shutdown() +{ + LogPrint("zmq", "zmq: Shutdown notification interface\n"); + if (pcontext) + { + for (std::list::iterator i = notifiers.begin(); i != notifiers.end(); ++i) + { + CZMQAbstractNotifier *notifier = *i; + LogPrint("zmq", " Shutdown notifier %s at %s\n", notifier->GetType(), notifier->GetAddress()); + notifier->Shutdown(); + } + zmq_ctx_destroy(pcontext); + + pcontext = 0; + } +} + +void CZMQNotificationInterface::UpdatedBlockTip(const CBlockIndex *pindex) +{ + for (std::list::iterator i = notifiers.begin(); i != notifiers.end();) + { + CZMQAbstractNotifier *notifier = *i; + if (notifier->NotifyBlock(pindex)) + { + i++; + } + else + { + notifier->Shutdown(); + i = notifiers.erase(i); + } + } +} + +void CZMQNotificationInterface::SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock, int txIndex) +{ + for (std::list::iterator i = notifiers.begin(); i != notifiers.end();) + { + CZMQAbstractNotifier *notifier = *i; + if (notifier->NotifyTransaction(ptx)) + { + i++; + } + else + { + notifier->Shutdown(); + i = notifiers.erase(i); + } + } +} diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h new file mode 100644 index 00000000..650bb7e5 --- /dev/null +++ b/src/zmq/zmqnotificationinterface.h @@ -0,0 +1,38 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H +#define BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H + +#include "validationinterface.h" +#include +#include + +class CBlockIndex; +class CZMQAbstractNotifier; + +class CZMQNotificationInterface : public CValidationInterface +{ +public: + virtual ~CZMQNotificationInterface(); + + static CZMQNotificationInterface *CreateWithArguments(const std::map &args); + +protected: + bool Initialize(); + void Shutdown(); + + // CValidationInterface + void SyncTransaction(const CTransactionRef &ptx, const CBlock *pblock, int txIndex = -1); + void UpdatedBlockTip(const CBlockIndex *pindex); + +private: + CZMQNotificationInterface(); + + void *pcontext; + std::list notifiers; +}; + +#endif // BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp new file mode 100644 index 00000000..9a111e96 --- /dev/null +++ b/src/zmq/zmqpublishnotifier.cpp @@ -0,0 +1,177 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "zmqpublishnotifier.h" +//#include "blockstorage/blockstorage.h" +//#include "chainparams.h" +#include "main.h" +#include "util/util.h" + +static std::multimap mapPublishNotifiers; + +// Internal function to send multipart message +static int zmq_send_multipart(void *sock, const void *data, size_t size, ...) +{ + va_list args; + va_start(args, size); + + while (1) + { + zmq_msg_t msg; + + int rc = zmq_msg_init_size(&msg, size); + if (rc != 0) + { + zmqError("Unable to initialize ZMQ msg"); + va_end(args); + return -1; + } + + void *buf = zmq_msg_data(&msg); + memcpy(buf, data, size); + + data = va_arg(args, const void *); + + rc = zmq_msg_send(&msg, sock, data ? ZMQ_SNDMORE : 0); + if (rc == -1) + { + zmqError("Unable to send ZMQ msg"); + zmq_msg_close(&msg); + va_end(args); + return -1; + } + + zmq_msg_close(&msg); + + if (!data) + break; + + size = va_arg(args, size_t); + } + va_end(args); + return 0; +} + +bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) +{ + assert(!psocket); + + // check if address is being used by other publish notifier + std::multimap::iterator i = mapPublishNotifiers.find(address); + + if (i == mapPublishNotifiers.end()) + { + psocket = zmq_socket(pcontext, ZMQ_PUB); + if (!psocket) + { + zmqError("Failed to create socket"); + return false; + } + + int rc = zmq_bind(psocket, address.c_str()); + if (rc != 0) + { + zmqError("Failed to bind address"); + return false; + } + + // register this notifier for the address, so it can be reused for other publish notifier + mapPublishNotifiers.insert(std::make_pair(address, this)); + return true; + } + else + { + LogPrint("zmq", "zmq: Reusing socket for address %s\n", address); + + psocket = i->second->psocket; + mapPublishNotifiers.insert(std::make_pair(address, this)); + + return true; + } +} + +void CZMQAbstractPublishNotifier::Shutdown() +{ + assert(psocket); + + int count = mapPublishNotifiers.count(address); + + // remove this notifier from the list of publishers using this address + typedef std::multimap::iterator iterator; + std::pair iterpair = mapPublishNotifiers.equal_range(address); + + for (iterator it = iterpair.first; it != iterpair.second; ++it) + { + if (it->second == this) + { + mapPublishNotifiers.erase(it); + break; + } + } + + if (count == 1) + { + LogPrint("zmq", "Close socket at address %s\n", address); + int linger = 0; + zmq_setsockopt(psocket, ZMQ_LINGER, &linger, sizeof(linger)); + zmq_close(psocket); + } + + psocket = 0; +} + +bool CZMQPublishHashBlockNotifier::NotifyBlock(const CBlockIndex *pindex) +{ + uint256 hash = pindex->GetBlockHash(); + LogPrint("zmq", "zmq: Publish hashblock %s\n", hash.GetHex()); + char data[32]; + for (unsigned int i = 0; i < 32; i++) + data[31 - i] = hash.begin()[i]; + int rc = zmq_send_multipart(psocket, "hashblock", 9, data, 32, 0); + return rc == 0; +} + +bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransactionRef &ptx) +{ + uint256 hash = ptx->GetHash(); + LogPrint("zmq", "zmq: Publish hashtx %s\n", hash.GetHex()); + char data[32]; + for (unsigned int i = 0; i < 32; i++) + data[31 - i] = hash.begin()[i]; + int rc = zmq_send_multipart(psocket, "hashtx", 6, data, 32, 0); + return rc == 0; +} + +bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) +{ + LogPrint("zmq", "zmq: Publish rawblock %s\n", pindex->GetBlockHash().GetHex()); + + const Consensus::Params &consensusParams = pnetMan->getActivePaymentNetwork()->GetConsensus(); + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + { + LOCK(cs_main); + CBlock block; + if (!ReadBlockFromDisk(block, pindex, consensusParams)) + { + zmqError("Can't read block from disk"); + return false; + } + + ss << block; + } + + int rc = zmq_send_multipart(psocket, "rawblock", 8, &(*ss.begin()), ss.size(), 0); + return rc == 0; +} + +bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransactionRef &ptx) +{ + uint256 hash = ptx->GetHash(); + LogPrint("zmq", "zmq: Publish rawtx %s\n", hash.GetHex()); + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << *ptx; + int rc = zmq_send_multipart(psocket, "rawtx", 5, &(*ss.begin()), ss.size(), 0); + return rc == 0; +} diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h new file mode 100644 index 00000000..d9e88e38 --- /dev/null +++ b/src/zmq/zmqpublishnotifier.h @@ -0,0 +1,44 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H +#define BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H + +#include "zmqabstractnotifier.h" + +class CBlockIndex; + +class CZMQAbstractPublishNotifier : public CZMQAbstractNotifier +{ +public: + bool Initialize(void *pcontext); + void Shutdown(); +}; + +class CZMQPublishHashBlockNotifier : public CZMQAbstractPublishNotifier +{ +public: + bool NotifyBlock(const CBlockIndex *pindex); +}; + +class CZMQPublishHashTransactionNotifier : public CZMQAbstractPublishNotifier +{ +public: + bool NotifyTransaction(const CTransactionRef &ptx); +}; + +class CZMQPublishRawBlockNotifier : public CZMQAbstractPublishNotifier +{ +public: + bool NotifyBlock(const CBlockIndex *pindex); +}; + +class CZMQPublishRawTransactionNotifier : public CZMQAbstractPublishNotifier +{ +public: + bool NotifyTransaction(const CTransactionRef &ptx); +}; + +#endif // BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H From a0b622d42aeba6b95ba2f2ee48e0b9e35aac75d9 Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Wed, 27 Feb 2019 03:08:43 -0500 Subject: [PATCH 10/46] lower minimum required peers to generate a block from 6 to 4 (#133) --- src/blockgeneration/blockgeneration.h | 2 +- src/blockgeneration/miner.cpp | 2 +- src/blockgeneration/minter.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/blockgeneration/blockgeneration.h b/src/blockgeneration/blockgeneration.h index 7f0821e9..6ba539f1 100644 --- a/src/blockgeneration/blockgeneration.h +++ b/src/blockgeneration/blockgeneration.h @@ -27,7 +27,7 @@ static const bool DEFAULT_GENERATE = false; static const bool DEFAULT_PRINTPRIORITY = false; - +static const uint64_t DEFAULT_MIN_BLOCK_GEN_PEERS = 4; struct CBlockTemplate { CBlock block; diff --git a/src/blockgeneration/miner.cpp b/src/blockgeneration/miner.cpp index e90017b3..e4a8fd6b 100644 --- a/src/blockgeneration/miner.cpp +++ b/src/blockgeneration/miner.cpp @@ -445,7 +445,7 @@ void EccMiner(CWallet *pwallet) if (shutdown_threads.load()) return; } - while (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) < 6 || + while (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) < DEFAULT_MIN_BLOCK_GEN_PEERS || pnetMan->getChainActive()->IsInitialBlockDownload() || pwallet->IsLocked()) { MilliSleep(1000); diff --git a/src/blockgeneration/minter.cpp b/src/blockgeneration/minter.cpp index efbc1be5..b85229c8 100644 --- a/src/blockgeneration/minter.cpp +++ b/src/blockgeneration/minter.cpp @@ -352,7 +352,7 @@ void EccMinter(CWallet *pwallet) if (shutdown_threads.load()) return; } - while (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) < 6 || + while (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) < DEFAULT_MIN_BLOCK_GEN_PEERS || pnetMan->getChainActive()->IsInitialBlockDownload() || pwallet->IsLocked()) { MilliSleep(1000); From d957918e68a46ac74854e1df0f53e694f0d933cf Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Wed, 27 Feb 2019 03:38:27 -0500 Subject: [PATCH 11/46] use the user configured maxTxFee as the absurd fee limit for transactions with a version of 1 (#135) --- src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index 194f6ea5..494e8196 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -737,7 +737,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, dFreeCount += nSize; } - if (fRejectAbsurdFee && tx.nVersion == 1 && nFees > ::minRelayTxFee.GetFee(nSize) * 10000) + if (fRejectAbsurdFee && tx.nVersion == 1 && nFees > maxTxFee) { LogPrintf("Absurdly-high-fee of %d for tx with version of 1 \n", nFees); return state.Invalid(false, REJECT_HIGHFEE, "absurdly-high-fee", From b8b5fb3af0d875b87bf45a839cbd3a3ecc0a0286 Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Wed, 27 Feb 2019 04:53:47 -0500 Subject: [PATCH 12/46] remove staking from getinfo. it is displayed in getmininginfo instead (#136) --- src/rpc/rpcmisc.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index 0a9115df..0bb57147 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -120,7 +120,6 @@ UniValue getinfo(const UniValue ¶ms, bool fHelp) obj.push_back(Pair("unlocked_until", nWalletUnlockTime)); } } - obj.push_back(Pair("staking", minerThreads != NULL)); obj.push_back(Pair("paytxfee", ValueFromAmount(payTxFee.GetFeePerK()))); obj.push_back(Pair("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK()))); obj.push_back(Pair("errors", GetWarnings("statusbar"))); From c158644fe68ed2d81f96cadf27c22de6fdb8e308 Mon Sep 17 00:00:00 2001 From: Greg Griffith Date: Thu, 28 Feb 2019 05:15:57 -0500 Subject: [PATCH 13/46] sendrawtx should relay the tx instead of directly pushing to each peer (#137) --- src/rpc/rpcrawtransaction.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 04e7085e..ff33c27a 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -46,6 +46,8 @@ #include +void RelayTransaction(const CTransaction &tx, CConnman &connman); + void ScriptPubKeyToJSON(const CScript &scriptPubKey, UniValue &out, bool fIncludeHex) { txnouttype type; @@ -913,7 +915,6 @@ UniValue sendrawtransaction(const UniValue ¶ms, bool fHelp) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } - CInv inv(MSG_TX, txid); - g_connman->ForEachNode([&inv](CNode *pnode) { pnode->PushInventory(inv); }); + RelayTransaction(ttx, *g_connman); return txid.GetHex(); } From 90ff369d8502ce8328ea1d75b252958d7df38ff6 Mon Sep 17 00:00:00 2001 From: Alton Jensen Date: Wed, 20 Mar 2019 23:59:48 -0600 Subject: [PATCH 14/46] add linearize & update Dockerfile to work better (#128) --- Dockerfile | 26 ++--- contrib/linearize/README.md | 2 + contrib/linearize/example-linearize.cfg | 12 +++ contrib/linearize/linearize.py | 132 ++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 contrib/linearize/README.md create mode 100644 contrib/linearize/example-linearize.cfg create mode 100755 contrib/linearize/linearize.py diff --git a/Dockerfile b/Dockerfile index 0dff878c..b865cee3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,28 @@ ################################## Notes ################################## # to build: -# docker build --no-cache -t eccoind . -# (--no-cache is required or else it won't pull latest updates from github) +# docker build -t eccoin . # # to run: -# docker run -p 19118:19118 eccoind +# docker run -p 19118:19118 eccoin # -# to run with a mounted directory for ~/.eccoind: -# docker run -p 19118:19118 -v /path/to/a/local/directory:/root/.eccoin eccoind +# to run with a mounted directory for ~/.eccoin: +# docker run -p 19118:19118 -v /path/to/a/local/directory:/root/.eccoin eccoin # ############################################################################# -FROM ubuntu:16.04 +FROM ubuntu:18.04 -MAINTAINER Alton Jensen version: 0.1 +MAINTAINER Alton Jensen version: 0.2 -RUN apt-get update && apt-get install -y libdb-dev libdb++-dev build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libminiupnpc-dev libzmq3-dev git unzip wget +#install necessary packages +RUN apt-get update && apt-get install -y libdb-dev libdb++-dev build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libminiupnpc-dev libzmq3-dev software-properties-common +RUN add-apt-repository ppa:bitcoin/bitcoin && apt-get update && apt-get install -y libdb4.8-dev libdb4.8++-dev RUN apt-get clean && rm -rf /var/lib/apt/lists/* -#build from latest refactor12 branch code -RUN git clone https://github.com/Greg-Griffith/ECCoin.git && cd ECCoin/src/ && make -f makefile.unix +#build from local copy of codebase +COPY . /eccoin/ +RUN cd eccoin && ./autogen.sh && ./configure && make +#final prep work to run daemon RUN mkdir /root/.eccoin/ - -CMD ["/ECCoin/src/ECCoind","-listen","-upnp"] +CMD ["/eccoin/src/eccoind","-listen","-upnp"] diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md new file mode 100644 index 00000000..70b9f034 --- /dev/null +++ b/contrib/linearize/README.md @@ -0,0 +1,2 @@ +### Linearize ### +Construct a linear, no-fork, best version of the blockchain. \ No newline at end of file diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg new file mode 100644 index 00000000..07c01d97 --- /dev/null +++ b/contrib/linearize/example-linearize.cfg @@ -0,0 +1,12 @@ +# eccoind RPC settings +rpcuser=someuser +rpcpassword=somepassword +host=127.0.0.1 +port=19119 + +# bootstrap.dat settings +netmagic=cef1dbfa +#max_height=279000 +output=bootstrap.dat + + diff --git a/contrib/linearize/linearize.py b/contrib/linearize/linearize.py new file mode 100755 index 00000000..650f7d36 --- /dev/null +++ b/contrib/linearize/linearize.py @@ -0,0 +1,132 @@ +#!/usr/bin/python +# +# linearize.py: Construct a linear, no-fork, best version of the blockchain. +# +# +# Copyright (c) 2013 The Bitcoin developers +# Distributed under the MIT/X11 software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +import json +import struct +import re +import base64 +import httplib +import sys + +ERR_SLEEP = 15 +MAX_NONCE = 1000000L + +settings = {} + +class BitcoinRPC: + OBJID = 1 + + def __init__(self, host, port, username, password): + authpair = "%s:%s" % (username, password) + self.authhdr = "Basic %s" % (base64.b64encode(authpair)) + self.conn = httplib.HTTPConnection(host, port, False, 30) + def rpc(self, method, params=None): + self.OBJID += 1 + obj = { 'version' : '1.1', + 'method' : method, + 'id' : self.OBJID } + if params is None: + obj['params'] = [] + else: + obj['params'] = params + self.conn.request('POST', '/', json.dumps(obj), + { 'Authorization' : self.authhdr, + 'Content-type' : 'application/json' }) + + resp = self.conn.getresponse() + if resp is None: + print "JSON-RPC: no response" + return None + + body = resp.read() + resp_obj = json.loads(body) + if resp_obj is None: + print "JSON-RPC: cannot JSON-decode body" + return None + if 'error' in resp_obj and resp_obj['error'] != None: + return resp_obj['error'] + if 'result' not in resp_obj: + print "JSON-RPC: no result in object" + return None + + return resp_obj['result'] + def getblock(self, hash, verbose=True): + return self.rpc('getblock', [hash, verbose]) + def getblockhash(self, index): + return self.rpc('getblockhash', [index]) + +def getblock(rpc, settings, n): + hash = rpc.getblockhash(n) + hexdata = rpc.getblock(hash, False) + data = hexdata.decode('hex') + + return data + +def get_blocks(settings): + rpc = BitcoinRPC(settings['host'], settings['port'], + settings['rpcuser'], settings['rpcpassword']) + + outf = open(settings['output'], 'ab') + + for height in xrange(settings['min_height'], settings['max_height']+1): + data = getblock(rpc, settings, height) + + outhdr = settings['netmagic'] + outhdr += struct.pack(" Date: Fri, 29 Mar 2019 20:57:05 +0900 Subject: [PATCH 15/46] removes extra/redeclared variables in accepttomemorypoolworker --- src/main.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 494e8196..73bec0aa 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -717,12 +717,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool &pool, nFees); if (fLimitFree && nModifiedFees < ::minRelayTxFee.GetFee(nSize)) { - static CCriticalSection csFreeLimiter; static double dFreeCount; - static int64_t nLastTime; - int64_t nNow = GetTime(); - - LOCK(csFreeLimiter); // Use an exponentially decaying ~10-minute window: dFreeCount *= pow(1.0 - 1.0 / 600.0, (double)(nNow - nLastTime)); From ad24a745700cde17883b0f053d765ed92ffcbcd5 Mon Sep 17 00:00:00 2001 From: Griffith Date: Sun, 31 Mar 2019 08:07:22 +0900 Subject: [PATCH 16/46] fix issue where nstart was not being incremented (#142) --- src/net/net.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/net/net.cpp b/src/net/net.cpp index 48d4c486..2abe4eb3 100644 --- a/src/net/net.cpp +++ b/src/net/net.cpp @@ -1592,6 +1592,8 @@ void ThreadMapPort() while (nStart <= nFinish && upnp_thread_shutdown.load() == false) { MilliSleep(10000); + // this is more performant than calling GetTime() multipe times + nStart += 10000; } if (upnp_thread_shutdown.load()) { From 123cf26b379f1f26078c1fe53dec57925710fb0d Mon Sep 17 00:00:00 2001 From: Griffith Date: Sat, 6 Apr 2019 00:06:19 +0900 Subject: [PATCH 17/46] Fix getheaders spam for faster IBD (#143) * move print statement into proper log category * only process headers if we dont already have them * if we get a header with no prev, still apply dos but re-request headers * adjust txpropagate.py test to fix random timeout --- qa/rpc-tests/txpropagate.py | 3 +-- src/net/messages.cpp | 26 ++++++++++++++++++++++++-- src/net/net.h | 2 +- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/qa/rpc-tests/txpropagate.py b/qa/rpc-tests/txpropagate.py index 5e687042..dc026b9d 100755 --- a/qa/rpc-tests/txpropagate.py +++ b/qa/rpc-tests/txpropagate.py @@ -39,8 +39,7 @@ def run_test(self): # generate non-empty blocks on the mining node to get a balance for x in range(0, 50): self.nodes[0].generate(1) - - self.sync_blocks() + self.sync_blocks() self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 100) time.sleep(1) diff --git a/src/net/messages.cpp b/src/net/messages.cpp index 5096c7ec..4da88634 100644 --- a/src/net/messages.cpp +++ b/src/net/messages.cpp @@ -1327,8 +1327,8 @@ bool static ProcessMessage(CNode *pfrom, // later (within the same cs_main lock, though). MarkBlockAsInFlight(pfrom->GetId(), inv.hash, chainparams.GetConsensus()); } - LogPrintf("getheaders (%d) %s to peer=%d\n", pnetMan->getChainActive()->pindexBestHeader->nHeight, - inv.hash.ToString(), pfrom->id); + LogPrint("net", "getheaders (%d) %s to peer=%d\n", + pnetMan->getChainActive()->pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); } } else @@ -1730,6 +1730,21 @@ bool static ProcessMessage(CNode *pfrom, return true; } + // check if these are duplicate headers + uint256 hash = headers.front().GetHash(); + CBlockIndex *_pindex = pnetMan->getChainActive()->LookupBlockIndex(hash); + if (hash != chainparams.GetConsensus().hashGenesisBlock) + { + if (_pindex) + { + if (_pindex->nStatus & BLOCK_FAILED_MASK) + { + return error("invalid header received"); + } + return true; + } + } + CBlockIndex *pindexLast = NULL; for (const CBlockHeader &header : headers) { @@ -1745,7 +1760,14 @@ bool static ProcessMessage(CNode *pfrom, if (state.IsInvalid(nDoS)) { if (nDoS > 0) + { Misbehaving(pfrom->GetId(), nDoS, state.GetRejectReason()); + } + if (state.GetRejectReason() == "bad-prevblk") + { + connman.PushMessage(pfrom, NetMsgType::GETHEADERS, + pnetMan->getChainActive()->chainActive.GetLocator(pindexLast), uint256()); + } return error("invalid header received"); } } diff --git a/src/net/net.h b/src/net/net.h index 070e9dd3..49404180 100644 --- a/src/net/net.h +++ b/src/net/net.h @@ -463,7 +463,7 @@ class CConnman CVectorWriter{SER_NETWORK, pnode->GetSendVersion(), data, 0, std::forward(args)...}; size_t nMessageSize = data.size(); size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrintf("sending %s (%d bytes) peer=%d\n", SanitizeString(sCommand.c_str()), nMessageSize, pnode->id); + LogPrint("net", "sending %s (%d bytes) peer=%d\n", SanitizeString(sCommand.c_str()), nMessageSize, pnode->id); std::vector serializedHeader; serializedHeader.reserve(CMessageHeader::HEADER_SIZE); From 4a3bffc3d03acae9454203a6bff9e148b15a7fff Mon Sep 17 00:00:00 2001 From: Griffith Date: Mon, 8 Apr 2019 12:30:37 +0900 Subject: [PATCH 18/46] add listaddresses rpc call to get a list of all used keys in the keypool (#145) --- src/rpc/rpcdump.cpp | 45 +++++++++++++++++++++++++++++++++++++++++++ src/rpc/rpcserver.cpp | 5 +++-- src/rpc/rpcserver.h | 1 + 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/rpc/rpcdump.cpp b/src/rpc/rpcdump.cpp index 6d5b700a..4808f7a5 100644 --- a/src/rpc/rpcdump.cpp +++ b/src/rpc/rpcdump.cpp @@ -526,3 +526,48 @@ UniValue dumpwallet(const UniValue ¶ms, bool fHelp) file.close(); return NullUniValue; } + +UniValue listaddresses(const UniValue ¶ms, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return NullUniValue; + + if (fHelp || params.size() != 0) + throw std::runtime_error("listaddresses \"filename\"\n" + "\nlists all used wallet keys in a human-readable format.\n" + "\nExamples:\n" + + HelpExampleCli("listaddresses", "") + HelpExampleRpc("listaddresses", "")); + + LOCK2(cs_main, pwalletMain->cs_wallet); + + EnsureWalletIsUnlocked(); + + std::map mapKeyBirth; + pwalletMain->GetKeyBirthTimes(mapKeyBirth); + + // sort time/key pairs + std::vector > vKeyBirth; + for (std::map::const_iterator it = mapKeyBirth.begin(); it != mapKeyBirth.end(); it++) + { + vKeyBirth.push_back(std::make_pair(it->second, it->first)); + } + mapKeyBirth.clear(); + std::sort(vKeyBirth.begin(), vKeyBirth.end()); + + UniValue ret(UniValue::VARR); + for (std::vector >::const_iterator it = vKeyBirth.begin(); it != vKeyBirth.end(); it++) + { + const CKeyID &keyid = it->second; + std::string strAddr = CBitcoinAddress(keyid).ToString(); + CKey key; + if (pwalletMain->GetKey(keyid, key)) + { + if (pwalletMain->mapAddressBook.count(keyid)) + { + ret.push_back(strprintf("%s", strAddr)); + } + } + } + + return ret; +} diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 4949d412..2dbcf8a5 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -359,8 +359,9 @@ static const CRPCCommand vRPCCommands[] = { /* Wallet */ {"wallet", "addmultisigaddress", &addmultisigaddress, true}, {"wallet", "backupwallet", &backupwallet, true}, {"wallet", "dumpprivkey", &dumpprivkey, true}, {"wallet", "dumpwallet", &dumpwallet, true}, - {"wallet", "encryptwallet", &encryptwallet, true}, {"wallet", "getbalance", &getbalance, false}, - {"wallet", "getnewaddress", &getnewaddress, true}, {"wallet", "getrawchangeaddress", &getrawchangeaddress, true}, + {"wallet", "listaddresses", &listaddresses, true}, {"wallet", "encryptwallet", &encryptwallet, true}, + {"wallet", "getbalance", &getbalance, false}, {"wallet", "getnewaddress", &getnewaddress, true}, + {"wallet", "getrawchangeaddress", &getrawchangeaddress, true}, {"wallet", "getreceivedbyaddress", &getreceivedbyaddress, false}, {"wallet", "gettransaction", &gettransaction, false}, {"wallet", "abandontransaction", &abandontransaction, false}, {"wallet", "getunconfirmedbalance", &getunconfirmedbalance, false}, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index a5bbee2d..db42f3f2 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -197,6 +197,7 @@ extern UniValue importprivkey(const UniValue ¶ms, bool fHelp); extern UniValue importaddress(const UniValue ¶ms, bool fHelp); extern UniValue importpubkey(const UniValue ¶ms, bool fHelp); extern UniValue dumpwallet(const UniValue ¶ms, bool fHelp); +extern UniValue listaddresses(const UniValue ¶ms, bool fHelp); extern UniValue importwallet(const UniValue ¶ms, bool fHelp); extern UniValue getgenerate(const UniValue ¶ms, bool fHelp); // in rpcmining.cpp From 03e0ab2ef9a699f9d66fc5d6ccbb81bf333d0f1c Mon Sep 17 00:00:00 2001 From: Henry Young <8385576+HenryYoung42@users.noreply.github.com> Date: Tue, 9 Apr 2019 10:12:49 +0800 Subject: [PATCH 19/46] Replaced deprecated std::auto_ptr with std::unique_ptr (#146) * Replaced deprecated std::auto_ptr with std::unique_ptr (per Bitcoin source) * Two shadow variable declaration fixes --- src/httpserver.cpp | 4 ++-- src/txmempool.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index cafdd270..c40935d7 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -276,7 +276,7 @@ static std::string RequestMethodString(HTTPRequest::RequestMethod m) /** HTTP request callback */ static void http_request_cb(struct evhttp_request *req, void *arg) { - std::auto_ptr hreq(new HTTPRequest(req)); + std::unique_ptr hreq(new HTTPRequest(req)); LogPrint("http", "Received a %s request for %s from %s\n", RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString()); @@ -317,7 +317,7 @@ static void http_request_cb(struct evhttp_request *req, void *arg) // Dispatch to worker thread if (i != iend) { - std::auto_ptr item(new HTTPWorkItem(hreq.release(), path, i->handler)); + std::unique_ptr item(new HTTPWorkItem(hreq.release(), path, i->handler)); assert(workQueue); if (workQueue->Enqueue(item.get())) item.release(); /* if true, queue took ownership */ diff --git a/src/txmempool.cpp b/src/txmempool.cpp index be0c390d..e538b908 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1198,8 +1198,8 @@ int CTxMemPool::Expire(int64_t time, std::vector &vCoinsToUncache) setEntries stage; for (txiter removeit : toremove) _CalculateDescendants(removeit, stage); - for (txiter it : stage) - for (const CTxIn &txin : it->GetTx().vin) + for (txiter iter : stage) + for (const CTxIn &txin : iter->GetTx().vin) vCoinsToUncache.push_back(txin.prevout); _RemoveStaged(stage); @@ -1333,8 +1333,8 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector *pvNoSpends if (pvNoSpendsRemaining) { txn.reserve(stage.size()); - BOOST_FOREACH (txiter it, stage) - txn.push_back(it->GetTx()); + BOOST_FOREACH (txiter iter, stage) + txn.push_back(iter->GetTx()); } _RemoveStaged(stage); if (pvNoSpendsRemaining) From f26afdfc21a9264a1f2c9a16bb2b72c52b2b2638 Mon Sep 17 00:00:00 2001 From: Griffith Date: Sat, 13 Apr 2019 17:29:30 +0900 Subject: [PATCH 20/46] add recursive shared mutex library and add it to makefiles (#149) --- src/Makefile.am | 5 +- src/Makefile.test.include | 5 + src/rsm/README.md | 42 ++++ src/rsm/recursive_shared_mutex.cpp | 313 ++++++++++++++++++++++++++ src/rsm/recursive_shared_mutex.h | 207 +++++++++++++++++ src/rsm/test/rsm_promotion_tests.cpp | 132 +++++++++++ src/rsm/test/rsm_simple_tests.cpp | 181 +++++++++++++++ src/rsm/test/rsm_starvation_tests.cpp | 98 ++++++++ src/rsm/test/test_cxx_rsm.cpp | 14 ++ src/rsm/test/test_cxx_rsm.h | 14 ++ src/rsm/test/timer.cpp | 24 ++ src/rsm/test/timer.h | 15 ++ 12 files changed, 1049 insertions(+), 1 deletion(-) create mode 100644 src/rsm/README.md create mode 100644 src/rsm/recursive_shared_mutex.cpp create mode 100644 src/rsm/recursive_shared_mutex.h create mode 100644 src/rsm/test/rsm_promotion_tests.cpp create mode 100644 src/rsm/test/rsm_simple_tests.cpp create mode 100644 src/rsm/test/rsm_starvation_tests.cpp create mode 100644 src/rsm/test/test_cxx_rsm.cpp create mode 100644 src/rsm/test/test_cxx_rsm.h create mode 100644 src/rsm/test/timer.cpp create mode 100644 src/rsm/test/timer.h diff --git a/src/Makefile.am b/src/Makefile.am index 0cec29a2..22359d61 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,4 +1,4 @@ -DIST_SUBDIRS = secp256k1 univalue +DIST_SUBDIRS = secp256k1 univalue rsm AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) AM_CXXFLAGS = $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS) @@ -10,6 +10,7 @@ BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/build $(BDB_CPPFLAGS) $(BOOST_CPPFL BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += -I$(srcdir)/univalue/include +BITCOIN_INCLUDES += -I$(srcdir)/rsm LIBBITCOIN_SERVER=libbitcoin_server.a LIBSECP256K1=secp256k1/libsecp256k1.la @@ -114,6 +115,7 @@ BITCOIN_CORE_H = \ rpc/rpcclient.h \ rpc/rpcprotocol.h \ rpc/rpcserver.h \ + rsm/recursive_shared_mutex.h \ script/interpreter.h \ script/script.h \ script/script_error.h \ @@ -197,6 +199,7 @@ libbitcoin_server_a_SOURCES = \ rpc/rpcnet.cpp \ rpc/rpcrawtransaction.cpp \ rpc/rpcserver.cpp \ + rsm/recursive_shared_mutex.cpp \ script/sigcache.cpp \ timedata.cpp \ torcontrol.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 6fbd9144..07bf1f7e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -94,6 +94,11 @@ BITCOIN_TESTS = \ test/uint256_tests.cpp \ test/univalue_tests.cpp +BITCOIN_TESTS += \ + rsm/test/rsm_promotion_tests.cpp \ + rsm/test/rsm_simple_tests.cpp \ + rsm/test/rsm_starvation_tests.cpp + BITCOIN_TESTS += \ wallet/test/wallet_tests.cpp # wallet/test/walletdb_tests.cpp \ diff --git a/src/rsm/README.md b/src/rsm/README.md new file mode 100644 index 00000000..9eaa9475 --- /dev/null +++ b/src/rsm/README.md @@ -0,0 +1,42 @@ +# cxx_recursive_shared_mutex + +A recursive shared mutex for C++ + +__Behaviorial Overview__ + +The recursive shared mutex has the following behavior implemented: +- This mutex has two levels of access, shared and exclusive. Multiple threads can own this mutex in shared mode but only one can own it in exclusive mode. +- A thread is considered to have ownership when it successfully calls either lock or try_lock (for either level of access). +- A thread may recursively call lock for ownership and must call a matching number of unlock calls to end ownership. +- A thread may call for shared ownership if it already has exclusive ownership without giving up exclusive ownership. +- There is internal tracking of how many times a thread locked for shared ownership. A thread can not unlock more times than it locked. Trying to do so will cause an assertion as this is a critical error somewhere in the locking logic. +- A thread may obtain exclusive ownership if no threads excluding itself have shared ownership by calling try_promotion(). Doing so while other threads have shared ownership will block until all other threads have released their shared ownership. Promoting ownership in this way will "jump the line" of other threads that waiting for exclusive ownership and will cause the thread with shared ownership to become the next thread to obtain exclusive ownership. To avoid deadlocks only one thread may attempt this ownership promotion at a time. If a thread has already done this and is currently waiting for promotion and a different thread tries to request promotion the try_promotion() call will return false. +- If a thread has exclusive ownership and checks if it has shared ownership we should should return true. + + +__NOTES__ + +- We use try_promotion() for promotions instead of lock() for two reasons: + - We want to be able to signal that we did not get the promotion. try_promotion() returns a boolean while lock() doesn't return anything. + - It is generally discouraged to have a lot of threads potentially calling and waiting for promotions because this creates a sort of race condition where the edited data set will be the same for the first thread to get promoted but all following threads aren't guaranteed to be editing the same data that was observed during shared ownership +- if the thread that has exclusive ownership got that ownership via promotion, another thread can not request a promotion to follow it. this prevents threads that use promotion to continuously "cut the line" for exclusive ownership. + + + + +__Development and Testing__ + +The `master` branch `rsm` folder should be stable at all times. To use rsm in your project just add the `rsm` folder to your project. +The `experimental` folder has changes that are in testing. To build the test suite, run Make. This should produce a binary named text_cxx_rsm. + + +__Requirements__ + +- C++14 or higher is required. +- Boost 1.55.0 or higher is recommended. + +__Upstream__ + +The upstream repository can be found at: https://github.com/Greg-Griffith/cxx_recursive_shared_mutex + +Initial development and feedback can be found at: https://github.com/BitcoinUnlimited/BitcoinUnlimited/pull/1591 diff --git a/src/rsm/recursive_shared_mutex.cpp b/src/rsm/recursive_shared_mutex.cpp new file mode 100644 index 00000000..390ae582 --- /dev/null +++ b/src/rsm/recursive_shared_mutex.cpp @@ -0,0 +1,313 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "recursive_shared_mutex.h" + +//////////////////////// +/// +/// Private Functions +/// + +bool recursive_shared_mutex::end_of_exclusive_ownership() +{ + return (_shared_while_exclusive_counter == 0 && _write_counter == 0); +} + +bool recursive_shared_mutex::check_for_write_lock(const std::thread::id &locking_thread_id) +{ + return (_write_owner_id == locking_thread_id); +} + +bool recursive_shared_mutex::check_for_write_unlock(const std::thread::id &locking_thread_id) +{ + if (_write_owner_id == locking_thread_id) + { + if (_shared_while_exclusive_counter == 0) + { +#ifdef DEBUG_ASSERTION + throw std::logic_error("can not unlock_shared more times than we locked for shared ownership while holding " + "exclusive ownership"); +#else + return true; +#endif + } + return true; + } + return false; +} + +bool recursive_shared_mutex::already_has_lock_shared(const std::thread::id &locking_thread_id) +{ + return (_read_owner_ids.find(locking_thread_id) != _read_owner_ids.end()); +} + +void recursive_shared_mutex::lock_shared_internal(const std::thread::id &locking_thread_id, const uint64_t &count) +{ + auto it = _read_owner_ids.find(locking_thread_id); + if (it == _read_owner_ids.end()) + { + _read_owner_ids.emplace(locking_thread_id, count); + } + else + { + it->second = it->second + count; + } +} + +void recursive_shared_mutex::unlock_shared_internal(const std::thread::id &locking_thread_id, const uint64_t &count) +{ + auto it = _read_owner_ids.find(locking_thread_id); + if (it == _read_owner_ids.end()) + { +#ifdef DEBUG_ASSERTION + throw std::logic_error("can not unlock_shared more times than we locked for shared ownership"); +#else + return; +#endif + } + it->second = it->second - count; + if (it->second == 0) + { + _read_owner_ids.erase(it); + } +} + +//////////////////////// +/// +/// Public Functions +/// + +void recursive_shared_mutex::lock() +{ + const std::thread::id &locking_thread_id = std::this_thread::get_id(); + std::unique_lock _lock(_mutex); + if (_write_owner_id == locking_thread_id) + { + _write_counter++; + } + else + { + // Wait until we can set the write-entered. + _read_gate.wait(_lock, [this] { return end_of_exclusive_ownership(); }); + + _write_counter++; + // Then wait until there are no more readers. + _write_gate.wait( + _lock, [this] { return _read_owner_ids.size() == 0 && _promotion_candidate_id == NON_THREAD_ID; }); + _write_owner_id = locking_thread_id; + } +} + +bool recursive_shared_mutex::try_promotion() +{ + const std::thread::id &locking_thread_id = std::this_thread::get_id(); + std::unique_lock _lock(_mutex); + + if (_write_owner_id == locking_thread_id) + { + _write_counter++; + return true; + } + // checking _write_owner_id might be redundant here with the mutex already being locked + // check if write_counter == 0 to ensure data consistency after promotion + else if (_promotion_candidate_id == NON_THREAD_ID) + { + _promotion_candidate_id = locking_thread_id; + // Then wait until there are no more readers. + _promotion_write_gate.wait(_lock, + [this] { return _read_owner_ids.size() == 1 && already_has_lock_shared(std::this_thread::get_id()); }); + _write_owner_id = locking_thread_id; + // it is possible that if we cut the line, another thread could have incremented the _write_counter + // already, so we should check this and decrement + save what they did + if (_write_counter != 0) + { + _write_counter_reserve = _write_counter; + _write_counter = 0; + } + // now increment the _write_counter for our own use + _write_counter++; + return true; + } + return false; +} + +bool recursive_shared_mutex::try_lock() +{ + const std::thread::id &locking_thread_id = std::this_thread::get_id(); + std::unique_lock _lock(_mutex, std::try_to_lock); + + if (_write_owner_id == locking_thread_id) + { + _write_counter++; + return true; + } + else if (_lock.owns_lock() && end_of_exclusive_ownership() && _read_owner_ids.size() == 0 && + _promotion_candidate_id == NON_THREAD_ID) + { + _write_counter++; + _write_owner_id = locking_thread_id; + return true; + } + return false; +} + +void recursive_shared_mutex::unlock() +{ + const std::thread::id &locking_thread_id = std::this_thread::get_id(); + std::lock_guard _lock(_mutex); + // you cannot unlock if you are not the write owner so check that here + // this might be redundant with the mutex being locked + if (_write_counter == 0 || _write_owner_id != locking_thread_id) + { +#ifdef DEBUG_ASSERTION + throw std::logic_error("unlock(standard logic) incorrectly called on a thread with no exclusive lock"); +#else + return; +#endif + } + if (_promotion_candidate_id != NON_THREAD_ID && _write_owner_id != _promotion_candidate_id) + { +#ifdef DEBUG_ASSERTION + throw std::logic_error("unlock(promotion logic) incorrectly called on a thread with no exclusive lock"); +#else + return; +#endif + } + if (_promotion_candidate_id != NON_THREAD_ID) + { + _write_counter--; + if (_write_counter == 0) + { +#ifdef DEBUG_ASSERTION + assert(_shared_while_exclusive_counter == 0); +#endif + if (_shared_while_exclusive_counter > 0) + { + lock_shared_internal(locking_thread_id, _shared_while_exclusive_counter); + _shared_while_exclusive_counter = 0; + } + // reset the write owner id back to a non thread id once we unlock all write locks + _write_owner_id = NON_THREAD_ID; + _promotion_candidate_id = NON_THREAD_ID; + // call notify_all() while mutex is held so that another thread can't + // lock and unlock the mutex then destroy *this before we make the call. + + // it is possible that if we cut the line, another thread could have incremented the _write_counter + // already, restore what they did + if (_write_counter_reserve != 0) + { + _write_counter = _write_counter_reserve; + _write_counter_reserve = 0; + } + + _read_gate.notify_all(); + } + } + else + { + _write_counter--; +#ifdef DEBUG_ASSERTION + assert(_write_counter_reserve == 0); +#endif + if (end_of_exclusive_ownership()) + { + // reset the write owner id back to a non thread id once we unlock all write locks + _write_owner_id = NON_THREAD_ID; + // call notify_all() while mutex is held so that another thread can't + // lock and unlock the mutex then destroy *this before we make the call. + + _read_gate.notify_all(); + } + } +} + +void recursive_shared_mutex::lock_shared() +{ + const std::thread::id &locking_thread_id = std::this_thread::get_id(); + std::unique_lock _lock(_mutex); + if (check_for_write_lock(locking_thread_id)) + { + _shared_while_exclusive_counter++; + return; + } + if (already_has_lock_shared(locking_thread_id)) + { + lock_shared_internal(locking_thread_id); + } + else + { + _read_gate.wait( + _lock, [this] { return end_of_exclusive_ownership() && _promotion_candidate_id == NON_THREAD_ID; }); + lock_shared_internal(locking_thread_id); + } +} + +bool recursive_shared_mutex::try_lock_shared() +{ + const std::thread::id &locking_thread_id = std::this_thread::get_id(); + std::unique_lock _lock(_mutex, std::try_to_lock); + if (check_for_write_lock(locking_thread_id)) + { + _shared_while_exclusive_counter++; + return true; + } + if (already_has_lock_shared(locking_thread_id)) + { + lock_shared_internal(locking_thread_id); + return true; + } + if (!_lock.owns_lock()) + { + return false; + } + if (end_of_exclusive_ownership() && _promotion_candidate_id == NON_THREAD_ID) + { + lock_shared_internal(locking_thread_id); + return true; + } + return false; +} + +void recursive_shared_mutex::unlock_shared() +{ + const std::thread::id &locking_thread_id = std::this_thread::get_id(); + std::lock_guard _lock(_mutex); + if (check_for_write_unlock(locking_thread_id)) + { + _shared_while_exclusive_counter--; + return; + } + if (_read_owner_ids.size() == 0) + { +#ifdef DEBUG_ASSERTION + throw std::logic_error("unlock_shared incorrectly called on a thread with no shared lock"); +#else + return; +#endif + } + unlock_shared_internal(locking_thread_id); + if (_promotion_candidate_id != NON_THREAD_ID) + { + if (_read_owner_ids.size() == 1 && already_has_lock_shared(_promotion_candidate_id)) + { + _promotion_write_gate.notify_one(); + } + else + { + _read_gate.notify_one(); + } + } + else if (_write_counter != 0 && _promotion_candidate_id == NON_THREAD_ID) + { + if (_read_owner_ids.size() == 0) + { + _write_gate.notify_one(); + } + else + { + _read_gate.notify_one(); + } + } +} diff --git a/src/rsm/recursive_shared_mutex.h b/src/rsm/recursive_shared_mutex.h new file mode 100644 index 00000000..192a7512 --- /dev/null +++ b/src/rsm/recursive_shared_mutex.h @@ -0,0 +1,207 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef _RECURSIVE_SHARED_MUTEX_H +#define _RECURSIVE_SHARED_MUTEX_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/** + * This mutex has two levels of access, shared and exclusive. Multiple threads can own this mutex in shared mode but + * only one can own it in exclusive mode. + * - A thread is considered to have ownership when it successfully calls either lock or try_lock. + * - A thread may recusively call lock for ownership and must call a matching number of unlock calls to end ownership. + * - A thread MAY call for shared ownership if it already has exclusive ownership. This should just increment the + * _shared_while_exclusive_counter instead of actually locking anything + * - A thread MAY obtain exclusive ownership if no threads excluding itself has shared ownership. (this might need to + * check for another write lock already queued up so we dont jump the line) + */ + + +static const std::thread::id NON_THREAD_ID = std::thread::id(); + +class recursive_shared_mutex +{ +protected: + // Only locked when accessing counters, ids, or waiting on condition variables. + std::mutex _mutex; + + // the read_gate is locked (blocked) when threads have write ownership + std::condition_variable _read_gate; + + // the write_gate is locked (blocked) when threads have read ownership or someone is waiting for promotion + std::condition_variable _write_gate; + + // // the write_gate is locked (blocked) when threads have read ownership + std::condition_variable _promotion_write_gate; + + // holds a list of owner ids that have shared ownership and the number of times they locked it + std::map _read_owner_ids; + + // holds the number of shared locks the thread with exclusive ownership has + // this is used to allow the thread with exclusive ownership to lock_shared + uint64_t _shared_while_exclusive_counter; + + // _write_counter tracks how many times exclusive ownership has been recursively locked + uint64_t _write_counter; + // _write_owner_id is the id of the thread with exclusive ownership + std::thread::id _write_owner_id; + // _promotion_candidate_id is the id of the thread waiting for a promotion + std::thread::id _promotion_candidate_id; + + // used to keep track of normal thread exclusive line if a thread has promoted + uint64_t _write_counter_reserve; + +private: + bool end_of_exclusive_ownership(); + bool check_for_write_lock(const std::thread::id &locking_thread_id); + bool check_for_write_unlock(const std::thread::id &locking_thread_id); + + bool already_has_lock_shared(const std::thread::id &locking_thread_id); + void lock_shared_internal(const std::thread::id &locking_thread_id, const uint64_t &count = 1); + void unlock_shared_internal(const std::thread::id &locking_thread_id, const uint64_t &count = 1); + +public: + recursive_shared_mutex() + { + _read_owner_ids.clear(); + _write_counter = 0; + _shared_while_exclusive_counter = 0; + _write_owner_id = NON_THREAD_ID; + _promotion_candidate_id = NON_THREAD_ID; + _write_counter_reserve = 0; + } + + ~recursive_shared_mutex() {} + recursive_shared_mutex(const recursive_shared_mutex &) = delete; + recursive_shared_mutex &operator=(const recursive_shared_mutex &) = delete; + + /** + * "Wait in line" for exclusive ownership of the mutex. + * + * This call is blocking when waiting for exclusive ownership. + * When exclusive ownership is obtained the id of the thread that made this call + * is stored in _write_ownder_id and _write_counter is incremeneted by 1. + * When called by a thread that already has exclusive ownership,t + * the _write_counter is incremeneted by 1 and call does not block. + * + * + * @param none + * @return none + */ + void lock(); + + /** + * Become "next in line" for exclusive ownership of the mutex if the promotion + * slot is not already occupied by another thread. + * + * When called by a thread that has shared ownership or no ownership, attempt to + * obtain the promotion slot. Only one thread can hold the promotion slot at a time. + * While promotion slot is obtained and waiting for exclusive ownership this + * call is blocking. + * When called by a thread that already has exclusive ownership, + * _write_counter is incremeneted by 1 and call does not block + * + * + * @param none + * @return: false on failure to be put in the promotion slot because + * it is already occupied by another thread. + * true when _write_counter has been incremented or exclusive ownership has been + * obtained + */ + bool try_promotion(); + + /** + * Attempt to claim exclusive ownership of the mutex if no threads + * have exclusive or shared ownership of the mutex including this one. + * + * This call never blocks. + * When called by a thread that already has exclusive ownership, + * _write_counter is incremeneted by 1 + * + * + * @param none + * @return: false on failure to obtain exclusive ownership. + * true when _write_counter has been incremented or exclusive ownership has been + * obtained + */ + bool try_lock(); + + /** + * Release 1 count of exclusive ownership. + * + * This call never blocks. + * When called by a thread that has exclusive ownership, either _write_counter is + * decremented by 1. When both write_counter and _shared_while_exclusive_counter + * are 0, exclusive ownership is released. + * + * + * @param none + * @return: none + */ + void unlock(); + + /** + * Attempt to claim shared ownership + * + * This call is blocking when waiting for shared ownership due to a thread having + * exclusive ownership. + * When shared ownership is obtained the id of the thread that made this call + * is stored in _read_owner_ids with a value of 1. Recursively locking for shared + * ownership increments the threads value in _read_owner_ids by 1. + * If this is called by a thread with exclusive ownership, increment the _shared_while_exclusive_counter + * by 1 instead of making an entry in _read_owner_ids + * + * + * @param none + * @return none + */ + void lock_shared(); + + /** + * Attempt to claim shared ownership of the mutex if no threads + * have exclusive ownership of the mutex. + * + * This call never blocks. + * When called by a thread that already has shared ownership, the threads + * _read_owner_ids value is incremeneted by 1 + * When called by a thread that has exclusive ownership, _shared_while_exclusive_counter is incremeneted by 1 + * + * + * @param none + * @return: false on failure to obtain shared ownership. + * true when the threads _read_owner_ids has been incremented or shared ownership has been + * obtained + */ + bool try_lock_shared(); + + /** + * Release 1 count of ownership + * + * This call never blocks. + * When called by a thread that has shared ownership, decrement the value of that thread in + * _read_owner_ids by 1. When that threads value reaches 0, remove it from _read_owner_ids signifying the + * end of shared ownership. + * When called by a thread with exclusive ownership decrement _shared_while_exclusive_counter by 1. + * + * + * @param none + * @return none + */ + void unlock_shared(); +}; + + +#endif // _RECURSIVE_SHARED_MUTEX_H diff --git a/src/rsm/test/rsm_promotion_tests.cpp b/src/rsm/test/rsm_promotion_tests.cpp new file mode 100644 index 00000000..9b7b61c5 --- /dev/null +++ b/src/rsm/test/rsm_promotion_tests.cpp @@ -0,0 +1,132 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "recursive_shared_mutex.h" +#include "test_cxx_rsm.h" +#include "timer.h" + +#include + +BOOST_FIXTURE_TEST_SUITE(rsm_promotion_tests, TestSetup) + +recursive_shared_mutex rsm; +std::vector rsm_guarded_vector; + +void helper_fail() { BOOST_CHECK_EQUAL(rsm.try_lock(), false); } +void helper_pass() +{ + BOOST_CHECK_EQUAL(rsm.try_lock(), true); + // unlock the try_lock + rsm.unlock(); +} + +// test locking shared while holding exclusive ownership +// we should require an equal number of unlock_shared for each lock_shared +BOOST_AUTO_TEST_CASE(rsm_lock_shared_while_exclusive_owner) +{ + // lock exclusive 3 times + rsm.lock(); + rsm.lock(); + rsm.lock(); + + // lock_shared twice + rsm.lock_shared(); + rsm.lock_shared(); + + // it should require 3 unlocks and 2 unlock_shareds to have another thread lock exclusive + + // dont unlock exclusive enough times + rsm.unlock(); + rsm.unlock(); + rsm.unlock_shared(); + rsm.unlock_shared(); + + // we expect helper_fail to fail + std::thread one(helper_fail); + one.join(); + + // relock + rsm.lock(); + rsm.lock(); + rsm.lock_shared(); + rsm.lock_shared(); + + // now try not unlocking shared enough times + rsm.unlock(); + rsm.unlock(); + rsm.unlock(); + rsm.unlock_shared(); + + // again we expect helper fail to fail + std::thread two(helper_fail); + two.join(); + + // unlock the last shared + rsm.unlock_shared(); + + // helper pass should pass now + std::thread three(helper_pass); + three.join(); +} + +void shared_only() +{ + rsm.lock_shared(); + // give time for theta to lock shared, eta to lock, and theta to ask for promotion + MilliSleep(4000); + rsm.unlock_shared(); +} + +void exclusive_only() +{ + rsm.lock(); + rsm_guarded_vector.push_back(4); + rsm.unlock(); +} + +void promoting_thread() +{ + rsm.lock_shared(); + // give time for eta to get in line to lock exclusive + MilliSleep(500); + bool promoted = rsm.try_promotion(); + BOOST_CHECK_EQUAL(promoted, true); + rsm_guarded_vector.push_back(7); + rsm.unlock(); + rsm.unlock_shared(); +} + +/* + * if a thread askes for a promotion while no other thread + * is currently asking for a promotion it will be put in line to grab the next + * exclusive lock even if another threads are waiting using lock() + * + * This test covers lock promotion from shared to exclusive. + * + */ + +BOOST_AUTO_TEST_CASE(rsm_try_promotion) +{ + // clear the data vector at test start + rsm_guarded_vector.clear(); + // test promotions + std::thread one(shared_only); + MilliSleep(250); + std::thread two(promoting_thread); + MilliSleep(250); + std::thread three(exclusive_only); + + one.join(); + two.join(); + three.join(); + + // 7 was added by the promoted thread, it should appear first in the vector + rsm.lock_shared(); + BOOST_CHECK_EQUAL(7, rsm_guarded_vector[0]); + BOOST_CHECK_EQUAL(4, rsm_guarded_vector[1]); + rsm.unlock_shared(); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/rsm/test/rsm_simple_tests.cpp b/src/rsm/test/rsm_simple_tests.cpp new file mode 100644 index 00000000..4c7845f2 --- /dev/null +++ b/src/rsm/test/rsm_simple_tests.cpp @@ -0,0 +1,181 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "recursive_shared_mutex.h" +#include "test_cxx_rsm.h" +#include "timer.h" + +#include + +BOOST_FIXTURE_TEST_SUITE(rsm_simple_tests, TestSetup) + +recursive_shared_mutex rsm; + +// basic lock and unlock tests +BOOST_AUTO_TEST_CASE(rsm_lock_unlock) +{ + // exclusive lock once + rsm.lock(); + +// try to unlock_shared an exclusive lock +// we should error here because exclusive locks can +// be not be unlocked by shared_ unlock method +#ifdef DEBUG_ASSERTION + BOOST_CHECK_THROW(rsm.unlock_shared(), std::logic_error); +#endif + + // unlock exclusive lock + + BOOST_CHECK_NO_THROW(rsm.unlock()); + + // exclusive lock once + rsm.lock(); + + // try to unlock exclusive lock + BOOST_CHECK_NO_THROW(rsm.unlock()); + +#ifdef DEBUG_ASSERTION + // try to unlock exclusive lock more times than we locked + BOOST_CHECK_THROW(rsm.unlock(), std::logic_error); +#endif + + // test complete +} + +// basic lock_shared and unlock_shared tests +BOOST_AUTO_TEST_CASE(rsm_lock_shared_unlock_shared) +{ + // lock shared + rsm.lock_shared(); + +#ifdef DEBUG_ASSERTION + // try to unlock exclusive when we only have shared + BOOST_CHECK_THROW(rsm.unlock(), std::logic_error); +#endif + + // unlock shared + rsm.unlock_shared(); + +#ifdef DEBUG_ASSERTION + // we should error here because we are unlocking more times than we locked + BOOST_CHECK_THROW(rsm.unlock_shared(), std::logic_error); +#endif + + // test complete +} + +// basic try_lock tests +BOOST_AUTO_TEST_CASE(rsm_try_lock) +{ + // try lock + rsm.try_lock(); + +#ifdef DEBUG_ASSERTION + // try to unlock_shared an exclusive lock + // we should error here because exclusive locks can + // be not be unlocked by shared_ unlock method + BOOST_CHECK_THROW(rsm.unlock_shared(), std::logic_error); +#endif + + // unlock exclusive lock + BOOST_CHECK_NO_THROW(rsm.unlock()); + + // try lock + rsm.try_lock(); + + // try to unlock exclusive lock + BOOST_CHECK_NO_THROW(rsm.unlock()); + +#ifdef DEBUG_ASSERTION + // try to unlock exclusive lock more times than we locked + BOOST_CHECK_THROW(rsm.unlock(), std::logic_error); +#endif + + // test complete +} + +// basic try_lock_shared tests +BOOST_AUTO_TEST_CASE(rsm_try_lock_shared) +{ + // try lock shared + rsm.try_lock_shared(); + +#ifdef DEBUG_ASSERTION + // unlock exclusive while we have shared lock + BOOST_CHECK_THROW(rsm.unlock(), std::logic_error); +#endif + + // unlock shared + BOOST_CHECK_NO_THROW(rsm.unlock_shared()); + +#ifdef DEBUG_ASSERTION + // we should error here because we are unlocking more times than we locked + BOOST_CHECK_THROW(rsm.unlock_shared(), std::logic_error); +#endif + + // test complete +} + +// test locking recursively 100 times for each lock type +BOOST_AUTO_TEST_CASE(rsm_100_lock_test) +{ + uint8_t i = 0; + // lock + while (i < 100) + { + BOOST_CHECK_NO_THROW(rsm.lock()); + ++i; + } + + while (i > 0) + { + BOOST_CHECK_NO_THROW(rsm.unlock()); + --i; + } + + // lock_shared + while (i < 100) + { + BOOST_CHECK_NO_THROW(rsm.lock_shared()); + ++i; + } + + while (i > 0) + { + BOOST_CHECK_NO_THROW(rsm.unlock_shared()); + --i; + } + + // try_lock + while (i < 100) + { + BOOST_CHECK_NO_THROW(rsm.try_lock()); + ++i; + } + + while (i > 0) + { + BOOST_CHECK_NO_THROW(rsm.unlock()); + --i; + } + + // try_lock_shared + while (i < 100) + { + BOOST_CHECK_NO_THROW(rsm.try_lock_shared()); + ++i; + } + + while (i > 0) + { + BOOST_CHECK_NO_THROW(rsm.unlock_shared()); + --i; + } + + // test complete +} + + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/rsm/test/rsm_starvation_tests.cpp b/src/rsm/test/rsm_starvation_tests.cpp new file mode 100644 index 00000000..e9a9bc28 --- /dev/null +++ b/src/rsm/test/rsm_starvation_tests.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "recursive_shared_mutex.h" +#include "test_cxx_rsm.h" +#include "timer.h" + +#include + +BOOST_FIXTURE_TEST_SUITE(rsm_starvation_tests, TestSetup) + +class rsm_watcher : public recursive_shared_mutex +{ +public: + size_t get_shared_owners_count() { return _read_owner_ids.size(); } +}; + +rsm_watcher rsm; +std::vector rsm_guarded_vector; + +void shared_only() +{ + rsm.lock_shared(); + // give time for theta to lock shared, eta to lock, and theta to ask for promotion + MilliSleep(2000); + rsm.unlock_shared(); +} + +void exclusive_only() +{ + rsm.lock(); + rsm_guarded_vector.push_back(4); + rsm.unlock(); +} + +void promoting_thread() +{ + rsm.lock_shared(); + // give time for eta to get in line to lock exclusive + MilliSleep(100); + bool promoted = rsm.try_promotion(); + BOOST_CHECK_EQUAL(promoted, true); + rsm_guarded_vector.push_back(7); + rsm.unlock(); + rsm.unlock_shared(); +} + +/* + * if a thread askes for a promotion while no other thread + * is currently asking for a promotion it will be put in line to grab the next + * exclusive lock even if another threads are waiting using lock() + * + * This test covers blocking of additional shared ownership aquisitions while + * a thread is waiting for promotion. + * + */ + +BOOST_AUTO_TEST_CASE(rsm_test_starvation) +{ + // clear the data vector at test start + rsm_guarded_vector.clear(); + + // start up intial shared thread to block immidiate exclusive grabbing + std::thread one(shared_only); + std::thread two(shared_only); + MilliSleep(50); + std::thread three(promoting_thread); + MilliSleep(50); + std::thread four(exclusive_only); + MilliSleep(75); + // we should always get 3 because five, six, and seven should be blocked by + // three promotion request leaving only one, two, and three with shared ownership + BOOST_CHECK_EQUAL(rsm.get_shared_owners_count(), 3); + std::thread five(shared_only); + BOOST_CHECK_EQUAL(rsm.get_shared_owners_count(), 3); + std::thread six(shared_only); + BOOST_CHECK_EQUAL(rsm.get_shared_owners_count(), 3); + std::thread seven(shared_only); + BOOST_CHECK_EQUAL(rsm.get_shared_owners_count(), 3); + + one.join(); + two.join(); + three.join(); + four.join(); + five.join(); + six.join(); + seven.join(); + + // 7 was added by the promoted thread, it should appear first in the vector + rsm.lock_shared(); + BOOST_CHECK_EQUAL(7, rsm_guarded_vector[0]); + BOOST_CHECK_EQUAL(4, rsm_guarded_vector[1]); + rsm.unlock_shared(); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/rsm/test/test_cxx_rsm.cpp b/src/rsm/test/test_cxx_rsm.cpp new file mode 100644 index 00000000..936ef797 --- /dev/null +++ b/src/rsm/test/test_cxx_rsm.cpp @@ -0,0 +1,14 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MODULE cxx_rsm + +#include "test_cxx_rsm.h" + +#include +#include + +BOOST_GLOBAL_FIXTURE( TestSetup ); diff --git a/src/rsm/test/test_cxx_rsm.h b/src/rsm/test/test_cxx_rsm.h new file mode 100644 index 00000000..b06b7325 --- /dev/null +++ b/src/rsm/test/test_cxx_rsm.h @@ -0,0 +1,14 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef TEST_CXX_RSM_H +#define TEST_CXX_RSM_H + +struct TestSetup { + TestSetup() { /* intentionally left blank */ } + ~TestSetup() { /* intentionally left blank */ } +}; + +#endif // TEST_CXX_RSM_H diff --git a/src/rsm/test/timer.cpp b/src/rsm/test/timer.cpp new file mode 100644 index 00000000..b8194b7e --- /dev/null +++ b/src/rsm/test/timer.cpp @@ -0,0 +1,24 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "timer.h" + +#include + +#include +#include +#include + +int64_t GetTimeMillis() +{ + std::chrono::milliseconds ms = std::chrono::duration_cast< std::chrono::milliseconds >( + std::chrono::system_clock::now().time_since_epoch()); + return std::chrono::duration(ms).count(); +} + +void MilliSleep(int64_t n) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(n)); +} diff --git a/src/rsm/test/timer.h b/src/rsm/test/timer.h new file mode 100644 index 00000000..ef1fd9ad --- /dev/null +++ b/src/rsm/test/timer.h @@ -0,0 +1,15 @@ +// Copyright (c) 2019 Greg Griffith +// Copyright (c) 2019 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef TIMER_H +#define TIMER_H + +#include +#include + +int64_t GetTimeMillis(); +void MilliSleep(int64_t n); + +#endif // TIMER_H From 087b5fcd8ce3f5a4624814afa2f2951b2814188f Mon Sep 17 00:00:00 2001 From: Griffith Date: Sat, 13 Apr 2019 19:09:21 +0900 Subject: [PATCH 21/46] enable more cpp unit tests (#150) [skip ci] --- src/Makefile.test.include | 26 +++++++++---------- src/test/policyestimator_tests.cpp | 40 +++++------------------------- src/test/script_P2SH_tests.cpp | 16 ++++++------ 3 files changed, 26 insertions(+), 56 deletions(-) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 07bf1f7e..96ca3e37 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -34,26 +34,14 @@ RAW_TEST_FILES = GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) # temporarily removed tests # -# test/base58_tests.cpp # # test/bip32_tests.cpp # # test/bloom_tests.cpp # -# test/checkblock_tests.cpp # -# test/compress_tests.cpp # # test/DoS_tests.cpp # # test/exploit_tests.cpp # -# test/hash_tests.cpp # -# test/key_tests.cpp # # test/main_tests.cpp # # test/mempool_tests.cpp # -# test/merkle_tests.cpp # # test/multisig_tests.cpp # # test/net_tests.cpp # -# test/prevector_tests.cpp # -# test/policyestimator_tests.cpp # -# test/rpc_tests.cpp # -# test/script_P2SH_tests.cpp # -# test/sighash_tests.cpp # -# test/transaction_tests.cpp # # test/txvalidationcache_tests.cpp # # test/versionbits_tests.cpp # # test/util_tests.cpp # @@ -72,25 +60,37 @@ BITCOIN_TESTS = \ test/addrman_tests.cpp \ test/allocator_tests.cpp \ test/base32_tests.cpp \ + test/base58_tests.cpp \ test/base64_tests.cpp \ test/bswap_tests.cpp \ + test/checkblock_tests.cpp \ test/coins_tests.cpp \ + test/compress_tests.cpp \ test/crypto_tests.cpp \ + test/dbwrapper_tests.cpp \ test/getarg_tests.cpp \ + test/hash_tests.cpp \ test/jsonutil.h \ test/jsonutil.cpp \ + test/key_tests.cpp \ test/limitedmap_tests.cpp \ - test/dbwrapper_tests.cpp \ + test/merkle_tests.cpp \ test/netbase_tests.cpp \ test/pmt_tests.cpp \ + test/policyestimator_tests.cpp \ + test/prevector_tests.cpp \ test/reverselock_tests.cpp \ + test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scriptnum_tests.cpp \ + test/script_P2SH_tests.cpp \ test/serialize_tests.cpp \ + test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ test/streams_tests.cpp \ test/timedata_tests.cpp \ + test/transaction_tests.cpp \ test/uint256_tests.cpp \ test/univalue_tests.cpp diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 38e225ea..51dd2995 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -46,8 +46,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) CScript garbage; for (unsigned int i = 0; i < 128; i++) garbage.push_back('X'); - CMutableTransaction tx; - std::list dummyConflicted; + CTransaction tx; + std::list dummyConflicted; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); @@ -55,7 +55,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) CFeeRate baseRate(basefee, ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); // Create a fake block - std::vector block; + std::vector block; int blocknum = 0; // Loop through 200 blocks @@ -87,7 +87,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { CTransaction btx; if (mpool.lookup(txHashes[9 - h].back(), btx)) - block.push_back(btx); + block.push_back(MakeTransactionRef(btx)); txHashes[9 - h].pop_back(); } } @@ -103,11 +103,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) BOOST_CHECK(mpool.estimateFee(3) == CFeeRate(0)); BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() < 8 * baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() > 8 * baseRate.GetFeePerK() - deltaFee); - int answerFound; - BOOST_CHECK(mpool.estimateSmartFee(1, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(3, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(4, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(8, &answerFound) == mpool.estimateFee(8) && answerFound == 8); } } @@ -122,7 +117,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int i = 1; i < 10; i++) { origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); - origPriEst.push_back(mpool.estimatePriority(i)); if (i > 1) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i - 1] <= origFeeEst[i - 2]); @@ -144,8 +138,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i - 1] + deltaFee); BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i - 1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i - 1] + deltaPri); - BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i - 1] - deltaPri); } @@ -170,14 +162,10 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum, dummyConflicted); } - int answerFound; for (int i = 1; i < 10; i++) { BOOST_CHECK( mpool.estimateFee(i) == CFeeRate(0) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i - 1] - deltaFee); - BOOST_CHECK(mpool.estimateSmartFee(i, &answerFound).GetFeePerK() > origFeeEst[answerFound - 1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) == -1 || mpool.estimatePriority(i) > origPriEst[i - 1] - deltaPri); - BOOST_CHECK(mpool.estimateSmartPriority(i, &answerFound) > origPriEst[answerFound - 1] - deltaPri); } // Mine all those transactions @@ -188,7 +176,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { CTransaction btx; if (mpool.lookup(txHashes[j].back(), btx)) - block.push_back(btx); + block.push_back(MakeTransactionRef(btx)); txHashes[j].pop_back(); } } @@ -197,7 +185,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int i = 1; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i - 1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i - 1] - deltaPri); } // Mine 200 more blocks where everything is mined every block @@ -217,7 +204,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) .FromTx(tx, &mpool)); CTransaction btx; if (mpool.lookup(hash, btx)) - block.push_back(btx); + block.push_back(MakeTransactionRef(btx)); } } mpool.removeForBlock(block, ++blocknum, dummyConflicted); @@ -226,21 +213,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int i = 1; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i - 1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i - 1] - deltaPri); - } - - // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee - // and that estimateSmartPriority returns essentially an infinite value - mpool.addUnchecked( - tx.GetHash(), entry.Fee(feeV[0][5]).Time(GetTime()).Priority(priV[1][5]).Height(blocknum).FromTx(tx, &mpool)); - // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[0][5] - mpool.TrimToSize(1); - BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[0][5]); - for (int i = 1; i < 10; i++) - { - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartPriority(i) == INF_PRIORITY); } } diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index 99f3266a..508f07c8 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -6,10 +6,13 @@ #include "script/script.h" #include "key.h" #include "keystore.h" +#include "main.h" #include "policy/policy.h" +#include "script/interpreter.h" #include "script/script_error.h" #include "script/sign.h" #include "test/test_bitcoin.h" +#include "wallet/wallet.h" #include @@ -40,7 +43,7 @@ static bool Verify(const CScript &scriptSig, const CScript &scriptPubKey, bool f txTo.vout[0].nValue = 1; return VerifyScript(scriptSig, scriptPubKey, fStrict ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE, - TransactionSignatureChecker(&txTo, 0, txFrom.vout[0].nValue), &err); + TransactionSignatureChecker(&txTo, 0), &err); } @@ -96,7 +99,7 @@ BOOST_AUTO_TEST_CASE(sign) txTo[i].vin[0].prevout.n = i; txTo[i].vin[0].prevout.hash = txFrom.GetHash(); txTo[i].vout[0].nValue = 1; - BOOST_CHECK_MESSAGE(IsMine(keystore, txFrom.vout[i].scriptPubKey, 0), strprintf("IsMine %d", i)); + BOOST_CHECK_MESSAGE(IsMine(keystore, txFrom.vout[i].scriptPubKey), strprintf("IsMine %d", i)); } for (int i = 0; i < 8; i++) { @@ -111,13 +114,8 @@ BOOST_AUTO_TEST_CASE(sign) txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; const CTxOut &output = txFrom.vout[txTo[i].vin[0].prevout.n]; -#ifdef BITCOIN_CASH - bool sigOK = CScriptCheck(nullptr, output.scriptPubKey, output.nValue, txTo[i], 0, - SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC | SCRIPT_ENABLE_SIGHASH_FORKID, false)(); -#else - bool sigOK = CScriptCheck(nullptr, output.scriptPubKey, output.nValue, txTo[i], 0, + bool sigOK = CScriptCheck(output.scriptPubKey, output.nValue, txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false)(); -#endif if (i == j) BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); else @@ -199,7 +197,7 @@ BOOST_AUTO_TEST_CASE(set) txTo[i].vin[0].prevout.hash = txFrom.GetHash(); txTo[i].vout[0].nValue = 1 * CENT; txTo[i].vout[0].scriptPubKey = inner[i]; - BOOST_CHECK_MESSAGE(IsMine(keystore, txFrom.vout[i].scriptPubKey, 0), strprintf("IsMine %d", i)); + BOOST_CHECK_MESSAGE(IsMine(keystore, txFrom.vout[i].scriptPubKey), strprintf("IsMine %d", i)); } for (int i = 0; i < 4; i++) { From 1142a1326a89ae2809dfa4ff134aa4a1daf0fc2d Mon Sep 17 00:00:00 2001 From: Griffith Date: Wed, 17 Apr 2019 00:34:06 +0900 Subject: [PATCH 22/46] add read write lock macros for rsm (#151) --- src/Makefile.am | 2 + src/sync.cpp | 17 ------- src/sync.h | 21 ++++++++ src/sync_rsm.cpp | 129 +++++++++++++++++++++++++++++++++++++++++++++++ src/sync_rsm.h | 67 ++++++++++++++++++++++++ 5 files changed, 219 insertions(+), 17 deletions(-) create mode 100644 src/sync_rsm.cpp create mode 100644 src/sync_rsm.h diff --git a/src/Makefile.am b/src/Makefile.am index 22359d61..7abf2cac 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -131,6 +131,7 @@ BITCOIN_CORE_H = \ support/cleanse.h \ support/pagelocker.h \ sync.h \ + sync_rsm.h \ threadgroup.h \ threadsafety.h \ timedata.h \ @@ -171,6 +172,7 @@ libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CP libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIC_FLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ sync.cpp \ + sync_rsm.cpp \ net/addrdb.cpp \ net/addrman.cpp \ blockgeneration/blockgeneration.cpp \ diff --git a/src/sync.cpp b/src/sync.cpp index 18a6e019..2ad54f9c 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -36,23 +36,6 @@ void PrintLockContention(const char *pszName, const char *pszFile, unsigned int #endif /* DEBUG_LOCKCONTENTION */ #ifdef DEBUG_LOCKORDER -#include - -#ifdef __linux__ -uint64_t getTid(void) -{ - // "native" thread id used so the number correlates with what is shown in gdb - pid_t tid = (pid_t)syscall(SYS_gettid); - return tid; -} -#else -uint64_t getTid(void) -{ - uint64_t tid = boost::lexical_cast(boost::this_thread::get_id()); - return tid; -} -#endif - // // Early deadlock detection. // Problem being solved: diff --git a/src/sync.h b/src/sync.h index c4cd8893..12fa4e02 100644 --- a/src/sync.h +++ b/src/sync.h @@ -54,6 +54,26 @@ LEAVE_CRITICAL_SECTION(mutex); // no RAII // // /////////////////////////////// +#ifdef DEBUG_LOCKORDER +#include + +#ifdef __linux__ +inline uint64_t getTid(void) +{ + // "native" thread id used so the number correlates with what is shown in gdb + pid_t tid = (pid_t)syscall(SYS_gettid); + return tid; +} +#else +inline uint64_t getTid(void) +{ + uint64_t tid = boost::lexical_cast(boost::this_thread::get_id()); + return tid; +} +#endif + +#endif // DEBUG_LOCKORDER + /** * Template mixin that adds -Wthread-safety locking * annotations to a subset of the mutex API. @@ -67,6 +87,7 @@ class LOCKABLE AnnotatedMixin : public PARENT bool try_lock() EXCLUSIVE_TRYLOCK_FUNCTION(true) { return PARENT::try_lock(); } }; + /** * Wrapped boost mutex: supports recursive locking, but no waiting * TODO: We should move away from using the recursive lock by default. diff --git a/src/sync_rsm.cpp b/src/sync_rsm.cpp new file mode 100644 index 00000000..b59fa7cb --- /dev/null +++ b/src/sync_rsm.cpp @@ -0,0 +1,129 @@ +#include "sync.h" +#include "sync_rsm.h" + +#include "util/logger.h" +#include "util/util.h" +#include "util/utilstrencodings.h" + +#include +#include + +#ifdef DEBUG_LOCKORDER +CRecursiveSharedCriticalSection::CRecursiveSharedCriticalSection() : name(NULL), exclusiveOwner(0) {} +CRecursiveSharedCriticalSection::CRecursiveSharedCriticalSection(const char *n) : name(n), exclusiveOwner(0) +{ +// print the address of named critical sections so they can be found in the mutrace output +#ifdef ENABLE_MUTRACE + if (name) + { + LogPrintf("CRecursiveSharedCriticalSection %s at %p\n", name, this); + fflush(stdout); + } +#endif +} + +CRecursiveSharedCriticalSection::~CRecursiveSharedCriticalSection() +{ +#ifdef ENABLE_MUTRACE + if (name) + { + LogPrintf("Destructing CRecursiveSharedCriticalSection %s\n", name); + fflush(stdout); + } +#endif + DeleteLock((void *)this); +} + +void CRecursiveSharedCriticalSection::lock_shared() +{ + uint64_t tid = getTid(); + // detect recursive locking + { + std::unique_lock lock(setlock); + auto alreadyLocked = sharedowners.find(tid); + if (alreadyLocked != sharedowners.end()) + { + LockInfoRecursive li = alreadyLocked->second; + LogPrintf("already locked at %s:%d, incrementing shared lock by 1 to %u\n", li.file, li.line, li.count++); + alreadyLocked->second.count++; + + } + else + { + sharedowners[tid] = LockInfoRecursive("", 0, 1); + } + } + internal_lock.lock_shared(); +} + +void CRecursiveSharedCriticalSection::unlock_shared() +{ + // detect recursive locking + uint64_t tid = getTid(); + { + std::unique_lock lock(setlock); + auto alreadyLocked = sharedowners.find(tid); + if (alreadyLocked == sharedowners.end()) + { + LockInfoRecursive li = alreadyLocked->second; + LogPrintf("never locked at %s:%d\n", li.file, li.line); + assert(alreadyLocked != sharedowners.end()); + } + alreadyLocked->second.count--; + if (alreadyLocked->second.count == 0) + { + sharedowners.erase(tid); + } + } + internal_lock.unlock_shared(); +} + +bool CRecursiveSharedCriticalSection::try_lock_shared() +{ + uint64_t tid = getTid(); + std::unique_lock lock(setlock); + + bool result = internal_lock.try_lock_shared(); + if (result) + { + auto alreadyLocked = sharedowners.find(tid); + if (alreadyLocked == sharedowners.end()) + { + sharedowners[tid] = LockInfoRecursive("", 0, 1); + } + else + { + alreadyLocked->second.count++; + } + } + return result; +} +void CRecursiveSharedCriticalSection::lock() +{ + internal_lock.lock(); + exclusiveOwner = getTid(); + exclusiveOwnerCount++; +} +void CRecursiveSharedCriticalSection::unlock() +{ + uint64_t tid = getTid(); + assert(exclusiveOwner == tid); + exclusiveOwnerCount--; + if (exclusiveOwnerCount == 0) + { + exclusiveOwner = 0; + } + internal_lock.unlock(); +} + +bool CRecursiveSharedCriticalSection::try_lock() +{ + bool result = internal_lock.try_lock(); + if (result) + { + exclusiveOwner = getTid(); + exclusiveOwnerCount++; + } + return result; +} +#endif diff --git a/src/sync_rsm.h b/src/sync_rsm.h new file mode 100644 index 00000000..7fa81d0c --- /dev/null +++ b/src/sync_rsm.h @@ -0,0 +1,67 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef SYNC_RSM_H +#define SYNC_RSM_H + +#include "rsm/recursive_shared_mutex.h" +#include "sync.h" +#include "threadsafety.h" +#include "util/util.h" +#include "util/utiltime.h" + +#include + +#ifndef DEBUG_LOCKORDER +typedef recursive_shared_mutex CRecursiveSharedCriticalSection; +/** Define a named, shared critical section that is named in debug builds. + Named critical sections are useful in conjunction with a lock analyzer to discover bottlenecks. */ +#define RSCRITSEC(x) CRecursiveSharedCriticalSection x +#else + +class CRecursiveSharedCriticalSection +{ +public: + class LockInfoRecursive + { + public: + const char *file; + unsigned int line; + uint32_t count; + LockInfoRecursive() : file(""), line(0), count(0) {} + LockInfoRecursive(const char *f, unsigned int l, uint32_t c) : file(f), line(l), count(c){} + }; + + recursive_shared_mutex internal_lock; + std::mutex setlock; + std::map sharedowners; + const char *name; + uint64_t exclusiveOwner; + uint64_t exclusiveOwnerCount; + CRecursiveSharedCriticalSection(const char *name); + CRecursiveSharedCriticalSection(); + ~CRecursiveSharedCriticalSection(); + void lock_shared(); + bool try_lock_shared(); + void unlock_shared(); + void lock(); + void unlock(); + bool try_lock(); +}; +#define RSCRITSEC(zzname) CRecursiveSharedCriticalSection zzname(#zzname) +#endif + +typedef CMutexReadLock CRecursiveReadBlock; +typedef CMutexLock CRecursiveWriteBlock; + +#define READLOCK_RECURSIVE(cs) CRecursiveReadBlock UNIQUIFY(readblock)(cs, #cs, __FILE__, __LINE__) +#define WRITELOCK_RECURSIVE(cs) CRecursiveWriteBlock UNIQUIFY(writeblock)(cs, #cs, __FILE__, __LINE__) +#define READLOCK2_RECURSIVE(cs1, cs2) \ + CRecursiveReadBlock UNIQUIFY(readblock1)(cs1, #cs1, __FILE__, __LINE__), UNIQUIFY(readblock2)(cs2, #cs2, __FILE__, __LINE__) +#define TRY_READ_LOCK_RECURSIVE(cs, name) CRecursiveReadBlock name(cs, #cs, __FILE__, __LINE__, true) + + +#endif // SYNC_RSM_H From 3236a84e1836544fede87efad3e93507b7aa4700 Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Sun, 21 Apr 2019 02:48:53 +0900 Subject: [PATCH 23/46] pull univalue updates from upstream --- src/univalue/README.md | 11 ++++--- src/univalue/configure.ac | 6 ++-- src/univalue/gen/gen.cpp | 4 +-- src/univalue/include/univalue.h | 28 ++++++---------- src/univalue/lib/univalue.cpp | 18 +++++------ src/univalue/lib/univalue_get.cpp | 3 +- src/univalue/lib/univalue_read.cpp | 46 ++++++++++++++------------- src/univalue/lib/univalue_utffilter.h | 4 +-- src/univalue/lib/univalue_write.cpp | 21 ++++++------ src/univalue/test/object.cpp | 16 ++++++++-- src/univalue/test/unitester.cpp | 17 +++++----- 11 files changed, 86 insertions(+), 88 deletions(-) diff --git a/src/univalue/README.md b/src/univalue/README.md index 36aa786a..46df8d31 100644 --- a/src/univalue/README.md +++ b/src/univalue/README.md @@ -12,6 +12,12 @@ an arbitrary depth. This class is aligned with the JSON standard, [RFC 7159](https://tools.ietf.org/html/rfc7159.html). +## Motivation + +UniValue is a reaction to json_spirit, seeking to minimize template +and memory use, providing a straightforward RAII class compatible with +link-time optimization and embedded uses. + ## Installation This project is a standard GNU @@ -25,8 +31,3 @@ $ ./configure $ make ``` -## Design - -UniValue provides a single dynamic RAII C++ object class, -and minimizes template use (contra json_spirit). - diff --git a/src/univalue/configure.ac b/src/univalue/configure.ac index 8298332a..f3131283 100644 --- a/src/univalue/configure.ac +++ b/src/univalue/configure.ac @@ -1,7 +1,7 @@ m4_define([libunivalue_major_version], [1]) m4_define([libunivalue_minor_version], [1]) -m4_define([libunivalue_micro_version], [3]) -m4_define([libunivalue_interface_age], [3]) +m4_define([libunivalue_micro_version], [4]) +m4_define([libunivalue_interface_age], [4]) # If you need a modifier for the version number. # Normally empty, but can be used to make "fixup" releases. m4_define([libunivalue_extraversion], []) @@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()]) -AC_INIT([univalue], [1.0.3], +AC_INIT([univalue], [1.0.4], [http://github.com/jgarzik/univalue/]) dnl make the compilation flags quiet unless V=1 is used diff --git a/src/univalue/gen/gen.cpp b/src/univalue/gen/gen.cpp index 17f36194..be3d4ccf 100644 --- a/src/univalue/gen/gen.cpp +++ b/src/univalue/gen/gen.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. // // To re-create univalue_escapes.h: @@ -12,8 +12,6 @@ #include #include "univalue.h" -using namespace std; - static bool initEscapes; static std::string escapes[256]; diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 6fb1af2e..225533ce 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #ifndef __UNIVALUE_H__ #define __UNIVALUE_H__ @@ -14,7 +14,6 @@ #include #include -#include // .get_int64() #include // std::pair class UniValue { @@ -26,14 +25,8 @@ class UniValue { typ = initialType; val = initialStr; } - UniValue(unsigned int val_) { - setInt((uint64_t)val_); - } - UniValue(unsigned long val_) { - setInt((uint64_t)val_); - } - UniValue(unsigned long long val_) { - setInt((uint64_t)val_); + UniValue(uint64_t val_) { + setInt(val_); } UniValue(int64_t val_) { setInt(val_); @@ -136,6 +129,10 @@ class UniValue { UniValue tmpVal(val_); return pushKV(key, tmpVal); } + bool pushKV(const std::string& key, bool val_) { + UniValue tmpVal((bool)val_); + return pushKV(key, tmpVal); + } bool pushKV(const std::string& key, int val_) { UniValue tmpVal((int64_t)val_); return pushKV(key, tmpVal); @@ -203,17 +200,10 @@ static inline std::pair Pair(const char *cKey, std::string return std::make_pair(key, uVal); } -static inline std::pair Pair(const char *cKey, unsigned long ulVal) -{ - std::string key(cKey); - UniValue uVal(ulVal); - return std::make_pair(key, uVal); -} - -static inline std::pair Pair(const char *cKey, unsigned long long ullVal) +static inline std::pair Pair(const char *cKey, uint64_t u64Val) { std::string key(cKey); - UniValue uVal(ullVal); + UniValue uVal(u64Val); return std::make_pair(key, uVal); } diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index d8ad7c4b..2b966ad8 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -10,8 +10,6 @@ #include "univalue.h" -using namespace std; - const UniValue NullUniValue; void UniValue::clear() @@ -37,15 +35,15 @@ bool UniValue::setBool(bool val_) return true; } -static bool validNumStr(const string& s) +static bool validNumStr(const std::string& s) { - string tokenVal; + std::string tokenVal; unsigned int consumed; enum jtokentype tt = getJsonToken(tokenVal, consumed, s.data(), s.data() + s.size()); return (tt == JTOK_NUMBER); } -bool UniValue::setNumStr(const string& val_) +bool UniValue::setNumStr(const std::string& val_) { if (!validNumStr(val_)) return false; @@ -58,7 +56,7 @@ bool UniValue::setNumStr(const string& val_) bool UniValue::setInt(uint64_t val_) { - ostringstream oss; + std::ostringstream oss; oss << val_; @@ -67,7 +65,7 @@ bool UniValue::setInt(uint64_t val_) bool UniValue::setInt(int64_t val_) { - ostringstream oss; + std::ostringstream oss; oss << val_; @@ -76,7 +74,7 @@ bool UniValue::setInt(int64_t val_) bool UniValue::setFloat(double val_) { - ostringstream oss; + std::ostringstream oss; oss << std::setprecision(16) << val_; @@ -85,7 +83,7 @@ bool UniValue::setFloat(double val_) return ret; } -bool UniValue::setStr(const string& val_) +bool UniValue::setStr(const std::string& val_) { clear(); typ = VSTR; diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp index eabcf2da..8b2da755 100644 --- a/src/univalue/lib/univalue_get.cpp +++ b/src/univalue/lib/univalue_get.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -11,6 +11,7 @@ #include #include #include +#include #include "univalue.h" diff --git a/src/univalue/lib/univalue_read.cpp b/src/univalue/lib/univalue_read.cpp index ae75cb46..c9cf717f 100644 --- a/src/univalue/lib/univalue_read.cpp +++ b/src/univalue/lib/univalue_read.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -8,8 +8,6 @@ #include "univalue.h" #include "univalue_utffilter.h" -using namespace std; - static bool json_isdigit(int ch) { return ((ch >= '0') && (ch <= '9')); @@ -42,7 +40,7 @@ static const char *hatoui(const char *first, const char *last, return first; } -enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, +enum jtokentype getJsonToken(std::string& tokenVal, unsigned int& consumed, const char *raw, const char *end) { tokenVal.clear(); @@ -114,7 +112,7 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, case '8': case '9': { // part 1: int - string numStr; + std::string numStr; const char *first = raw; @@ -174,7 +172,7 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, case '"': { raw++; // skip " - string valStr; + std::string valStr; JSONUTF8StringFilter writer(valStr); while (true) { @@ -255,9 +253,9 @@ bool UniValue::read(const char *raw, size_t size) clear(); uint32_t expectMask = 0; - vector stack; + std::vector stack; - string tokenVal; + std::string tokenVal; unsigned int consumed; enum jtokentype tok = JTOK_NONE; enum jtokentype last_tok = JTOK_NONE; @@ -267,7 +265,7 @@ bool UniValue::read(const char *raw, size_t size) tok = getJsonToken(tokenVal, consumed, raw, end); if (tok == JTOK_NONE || tok == JTOK_ERR) - return false; + goto return_fail; raw += consumed; bool isValueOpen = jsonTokenIsValue(tok) || @@ -275,33 +273,33 @@ bool UniValue::read(const char *raw, size_t size) if (expect(VALUE)) { if (!isValueOpen) - return false; + goto return_fail; clearExpect(VALUE); } else if (expect(ARR_VALUE)) { bool isArrValue = isValueOpen || (tok == JTOK_ARR_CLOSE); if (!isArrValue) - return false; + goto return_fail; clearExpect(ARR_VALUE); } else if (expect(OBJ_NAME)) { bool isObjName = (tok == JTOK_OBJ_CLOSE || tok == JTOK_STRING); if (!isObjName) - return false; + goto return_fail; } else if (expect(COLON)) { if (tok != JTOK_COLON) - return false; + goto return_fail; clearExpect(COLON); } else if (!expect(COLON) && (tok == JTOK_COLON)) { - return false; + goto return_fail; } if (expect(NOT_VALUE)) { if (isValueOpen) - return false; + goto return_fail; clearExpect(NOT_VALUE); } @@ -335,12 +333,12 @@ bool UniValue::read(const char *raw, size_t size) case JTOK_OBJ_CLOSE: case JTOK_ARR_CLOSE: { if (!stack.size() || (last_tok == JTOK_COMMA)) - return false; + goto return_fail; VType utyp = (tok == JTOK_OBJ_CLOSE ? VOBJ : VARR); UniValue *top = stack.back(); if (utyp != top->getType()) - return false; + goto return_fail; stack.pop_back(); clearExpect(OBJ_NAME); @@ -350,11 +348,11 @@ bool UniValue::read(const char *raw, size_t size) case JTOK_COLON: { if (!stack.size()) - return false; + goto return_fail; UniValue *top = stack.back(); if (top->getType() != VOBJ) - return false; + goto return_fail; setExpect(VALUE); break; @@ -363,7 +361,7 @@ bool UniValue::read(const char *raw, size_t size) case JTOK_COMMA: { if (!stack.size() || (last_tok == JTOK_COMMA) || (last_tok == JTOK_ARR_OPEN)) - return false; + goto return_fail; UniValue *top = stack.back(); if (top->getType() == VOBJ) @@ -437,15 +435,19 @@ bool UniValue::read(const char *raw, size_t size) } default: - return false; + goto return_fail; } } while (!stack.empty ()); /* Check that nothing follows the initial construct (parsed above). */ tok = getJsonToken(tokenVal, consumed, raw, end); if (tok != JTOK_NONE) - return false; + goto return_fail; return true; + +return_fail: + clear(); + return false; } diff --git a/src/univalue/lib/univalue_utffilter.h b/src/univalue/lib/univalue_utffilter.h index b4a9ddc0..c24ac58e 100644 --- a/src/univalue/lib/univalue_utffilter.h +++ b/src/univalue/lib/univalue_utffilter.h @@ -1,6 +1,6 @@ // Copyright 2016 Wladimir J. van der Laan // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #ifndef UNIVALUE_UTFFILTER_H #define UNIVALUE_UTFFILTER_H @@ -13,7 +13,7 @@ class JSONUTF8StringFilter { public: - JSONUTF8StringFilter(std::string &s): + explicit JSONUTF8StringFilter(std::string &s): str(s), is_valid(true), codepoint(0), state(0), surpair(0) { } diff --git a/src/univalue/lib/univalue_write.cpp b/src/univalue/lib/univalue_write.cpp index cf278359..db039fcb 100644 --- a/src/univalue/lib/univalue_write.cpp +++ b/src/univalue/lib/univalue_write.cpp @@ -1,18 +1,15 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include -#include #include #include "univalue.h" #include "univalue_escapes.h" -using namespace std; - -static string json_escape(const string& inS) +static std::string json_escape(const std::string& inS) { - string outS; + std::string outS; outS.reserve(inS.size() * 2); for (unsigned int i = 0; i < inS.size(); i++) { @@ -28,10 +25,10 @@ static string json_escape(const string& inS) return outS; } -string UniValue::write(unsigned int prettyIndent, - unsigned int indentLevel) const +std::string UniValue::write(unsigned int prettyIndent, + unsigned int indentLevel) const { - string s; + std::string s; s.reserve(1024); unsigned int modIndent = indentLevel; @@ -62,12 +59,12 @@ string UniValue::write(unsigned int prettyIndent, return s; } -static void indentStr(unsigned int prettyIndent, unsigned int indentLevel, string& s) +static void indentStr(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) { s.append(prettyIndent * indentLevel, ' '); } -void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, string& s) const +void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const { s += "["; if (prettyIndent) @@ -89,7 +86,7 @@ void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, s s += "]"; } -void UniValue::writeObject(unsigned int prettyIndent, unsigned int indentLevel, string& s) const +void UniValue::writeObject(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const { s += "{"; if (prettyIndent) diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 679cc9f1..a1684939 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -1,7 +1,7 @@ // Copyright (c) 2014 BitPay Inc. // Copyright (c) 2014-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -261,6 +261,12 @@ BOOST_AUTO_TEST_CASE(univalue_object) strKey = "temperature"; BOOST_CHECK(obj.pushKV(strKey, (double) 90.012)); + strKey = "moon"; + BOOST_CHECK(obj.pushKV(strKey, true)); + + strKey = "spoon"; + BOOST_CHECK(obj.pushKV(strKey, false)); + UniValue obj2(UniValue::VOBJ); BOOST_CHECK(obj2.pushKV("cat1", 9000)); BOOST_CHECK(obj2.pushKV("cat2", 12345)); @@ -268,7 +274,7 @@ BOOST_AUTO_TEST_CASE(univalue_object) BOOST_CHECK(obj.pushKVs(obj2)); BOOST_CHECK_EQUAL(obj.empty(), false); - BOOST_CHECK_EQUAL(obj.size(), 9); + BOOST_CHECK_EQUAL(obj.size(), 11); BOOST_CHECK_EQUAL(obj["age"].getValStr(), "100"); BOOST_CHECK_EQUAL(obj["first"].getValStr(), "John"); @@ -277,6 +283,8 @@ BOOST_AUTO_TEST_CASE(univalue_object) BOOST_CHECK_EQUAL(obj["time"].getValStr(), "3600"); BOOST_CHECK_EQUAL(obj["calories"].getValStr(), "12"); BOOST_CHECK_EQUAL(obj["temperature"].getValStr(), "90.012"); + BOOST_CHECK_EQUAL(obj["moon"].getValStr(), "1"); + BOOST_CHECK_EQUAL(obj["spoon"].getValStr(), ""); BOOST_CHECK_EQUAL(obj["cat1"].getValStr(), "9000"); BOOST_CHECK_EQUAL(obj["cat2"].getValStr(), "12345"); @@ -289,6 +297,8 @@ BOOST_AUTO_TEST_CASE(univalue_object) BOOST_CHECK(obj.exists("time")); BOOST_CHECK(obj.exists("calories")); BOOST_CHECK(obj.exists("temperature")); + BOOST_CHECK(obj.exists("moon")); + BOOST_CHECK(obj.exists("spoon")); BOOST_CHECK(obj.exists("cat1")); BOOST_CHECK(obj.exists("cat2")); @@ -302,6 +312,8 @@ BOOST_AUTO_TEST_CASE(univalue_object) objTypes["time"] = UniValue::VNUM; objTypes["calories"] = UniValue::VNUM; objTypes["temperature"] = UniValue::VNUM; + objTypes["moon"] = UniValue::VBOOL; + objTypes["spoon"] = UniValue::VBOOL; objTypes["cat1"] = UniValue::VNUM; objTypes["cat2"] = UniValue::VNUM; BOOST_CHECK(obj.checkObject(objTypes)); diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 2c37794a..89157142 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -17,8 +17,7 @@ #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) #endif -using namespace std; -string srcdir(JSON_TEST_SRC); +std::string srcdir(JSON_TEST_SRC); static bool test_failed = false; #define d_assert(expr) { if (!(expr)) { test_failed = true; fprintf(stderr, "%s failed\n", filename.c_str()); } } @@ -30,9 +29,9 @@ static std::string rtrim(std::string s) return s; } -static void runtest(string filename, const string& jdata) +static void runtest(std::string filename, const std::string& jdata) { - string prefix = filename.substr(0, 4); + std::string prefix = filename.substr(0, 4); bool wantPass = (prefix == "pass") || (prefix == "roun"); bool wantFail = (prefix == "fail"); @@ -56,19 +55,19 @@ static void runtest(string filename, const string& jdata) static void runtest_file(const char *filename_) { - string basename(filename_); - string filename = srcdir + "/" + basename; + std::string basename(filename_); + std::string filename = srcdir + "/" + basename; FILE *f = fopen(filename.c_str(), "r"); assert(f != NULL); - string jdata; + std::string jdata; char buf[4096]; while (!feof(f)) { int bread = fread(buf, 1, sizeof(buf), f); assert(!ferror(f)); - string s(buf, bread); + std::string s(buf, bread); jdata += s; } From 65f5279fcae4f8eb7d8341d84816ed8b3acfe5e6 Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Sun, 21 Apr 2019 03:02:12 +0900 Subject: [PATCH 24/46] add debug checks to univalue --- configure.ac | 3 +++ src/univalue/configure.ac | 14 ++++++++++++-- src/univalue/lib/univalue.cpp | 31 +++++++++++++++++++++++++------ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index fb769157..57fad230 100644 --- a/configure.ac +++ b/configure.ac @@ -1021,6 +1021,9 @@ unset PKG_CONFIG_LIBDIR PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" ac_configure_args="${ac_configure_args} --disable-shared --with-pic --with-bignum=no --enable-module-recovery --disable-jni" +if test "x$enable_debug" = xyes; then + ac_configure_args="${ac_configure_args} --enable-debug" +fi AC_CONFIG_SUBDIRS([src/secp256k1 src/univalue]) AC_OUTPUT diff --git a/src/univalue/configure.ac b/src/univalue/configure.ac index f3131283..59c51cd5 100644 --- a/src/univalue/configure.ac +++ b/src/univalue/configure.ac @@ -2,7 +2,7 @@ m4_define([libunivalue_major_version], [1]) m4_define([libunivalue_minor_version], [1]) m4_define([libunivalue_micro_version], [4]) m4_define([libunivalue_interface_age], [4]) -# If you need a modifier for the version number. +# If you need a modifier for the version number. # Normally empty, but can be used to make "fixup" releases. m4_define([libunivalue_extraversion], []) @@ -45,6 +45,17 @@ AC_SUBST(LIBUNIVALUE_AGE) LT_INIT LT_LANG([C++]) +# Enable debug +AC_ARG_ENABLE([debug], + [AS_HELP_STRING([--enable-debug], + [use debug compiler flags and macros (default is no)])], + [enable_debug=$enableval], + [enable_debug=no]) + +if test "x$enable_debug" = xyes; then + CPPFLAGS="$CPPFLAGS -DDEBUG" +fi + case $host in *mingw*) LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static" @@ -66,4 +77,3 @@ AC_CONFIG_FILES([ AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(BUILD_EXEEXT) AC_OUTPUT - diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 2b966ad8..1ad9f396 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or https://opensource.org/licenses/mit-license.php. +// file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include @@ -107,18 +107,28 @@ bool UniValue::setObject() bool UniValue::push_back(const UniValue& val_) { +#ifdef DEBUG + assert(typ == VARR); +#else if (typ != VARR) + { return false; - + } +#endif values.push_back(val_); return true; } bool UniValue::push_backV(const std::vector& vec) { +#ifdef DEBUG + assert(typ == VARR); +#else if (typ != VARR) + { return false; - + } +#endif values.insert(values.end(), vec.begin(), vec.end()); return true; @@ -132,9 +142,14 @@ void UniValue::__pushKV(const std::string& key, const UniValue& val_) bool UniValue::pushKV(const std::string& key, const UniValue& val_) { +#ifdef DEBUG + assert(typ == VOBJ); +#else if (typ != VOBJ) + { return false; - + } +#endif size_t idx; if (findKey(key, idx)) values[idx] = val_; @@ -145,9 +160,14 @@ bool UniValue::pushKV(const std::string& key, const UniValue& val_) bool UniValue::pushKVs(const UniValue& obj) { +#ifdef DEBUG + assert(typ == VOBJ && obj.typ == VOBJ); +#else if (typ != VOBJ || obj.typ != VOBJ) + { return false; - + } +#endif for (size_t i = 0; i < obj.keys.size(); i++) __pushKV(obj.keys[i], obj.values.at(i)); @@ -239,4 +259,3 @@ const UniValue& find_value(const UniValue& obj, const std::string& name) return NullUniValue; } - From 1dd4a682c8bdb9d306397ed20b01863571be9675 Mon Sep 17 00:00:00 2001 From: Griffith Date: Sun, 21 Apr 2019 22:38:52 +0900 Subject: [PATCH 25/46] define bitcoin, bit, and satoshi amounts (#155) --- src/amount.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/amount.h b/src/amount.h index aee9a141..d82b827a 100644 --- a/src/amount.h +++ b/src/amount.h @@ -28,8 +28,15 @@ typedef int64_t CAmount; -static const CAmount COIN = 1000000; -static const CAmount CENT = 10000; +// clang-format off + +static const CAmount BITCOIN = 100000000; +static const CAmount COIN = 1000000; // 1 ECC +static const CAmount CENT = 10000; +static const CAmount BIT = 100; +static const CAmount SATOSHI = 1; + +// clang-format on extern const std::string CURRENCY_UNIT; From d8a39d9ebbd7f26a9e59201b580f365c6af94f16 Mon Sep 17 00:00:00 2001 From: Griffith Date: Mon, 22 Apr 2019 01:24:16 +0900 Subject: [PATCH 26/46] require a c++14 compiler (#156) --- configure.ac | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/configure.ac b/configure.ac index 57fad230..0f7fe7b8 100644 --- a/configure.ac +++ b/configure.ac @@ -66,8 +66,8 @@ case $host in lt_cv_deplibs_check_method="pass_all" ;; esac -dnl Require C++11 compiler (no GNU extensions) -AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) +dnl Require C++14 compiler (no GNU extensions) +AX_CXX_COMPILE_STDCXX([14], [noext], [mandatory], [nodefault]) dnl Check if -latomic is required for CHECK_ATOMIC @@ -683,25 +683,6 @@ AX_BOOST_THREAD AX_BOOST_CHRONO -if test x$use_reduce_exports = xyes; then - AC_MSG_CHECKING([for working boost reduced exports]) - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$BOOST_CPPFLAGS $CPPFLAGS" - AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ - @%:@include - ]], [[ - #if BOOST_VERSION >= 104900 - // Everything is okay - #else - # error Boost version is too old - #endif - ]])],[ - AC_MSG_RESULT(yes) - ],[ - AC_MSG_ERROR([boost versions < 1.49 are known to be broken with reduced exports. Use --disable-reduce-exports.]) - ]) - CPPFLAGS="$TEMP_CPPFLAGS" -fi fi if test x$use_reduce_exports = xyes; then From 00a1686a3f51ef6f3afc55356a6ab92bad250756 Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Mon, 22 Apr 2019 01:28:58 +0900 Subject: [PATCH 27/46] update travis dist from trusty to xenail --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e0038a0..e4402b9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: required -dist: trusty +dist: xenial os: linux -language: generic +language: minimal cache: directories: - depends/built @@ -33,7 +33,7 @@ matrix: addons: apt: sources: - - llvm-toolchain-trusty-5.0 + - llvm-toolchain-xenial-5.0 packages: - clang-5.0 env: From 90d3fe2f482fca62e120d37b6d9bf2e4cbb79955 Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Mon, 22 Apr 2019 01:29:06 +0900 Subject: [PATCH 28/46] remove DEP_OPTS defines, eccoin doesnt use QT or have a no wallet option --- .travis.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4402b9d..fdc0c9c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,6 @@ env: - PYTHON_DEBUG=1 - WINEDEBUG=fixme-all - PPA=ppa:bitcoin-unlimited/bu-ppa - - PPA2=ppa:sickpig/boost matrix: include: @@ -42,43 +41,42 @@ matrix: - CXXFLAGS="-std=c++11" - HOST=x86_64-unknown-linux-gnu - RUN_TESTS=false - - PACKAGES="python3-zmq libzmq3-dev clang-format-3.8 build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils libboost-all-dev qtbase5-dev protobuf-compiler qttools5-dev-tools qttools5-dev" - - GOAL="install" BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 CC=clang-5.0 CXX=clang++-5.0 CPPFLAGS=-DDEBUG_LOCKORDER" + - PACKAGES="python3-zmq libzmq3-dev clang-format-3.8 build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils libboost-all-dev" + - GOAL="install" BITCOIN_CONFIG="--enable-zmq CC=clang-5.0 CXX=clang++-5.0 CPPFLAGS=-DDEBUG_LOCKORDER" #bitcoind - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=x86_64-unknown-linux-gnu PACKAGES="python3-zmq clang-format-3.8" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true RUN_FORMATTING_CHECK=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" + - HOST=x86_64-unknown-linux-gnu PACKAGES="python3-zmq clang-format-3.8" DEP_OPTS="NO_UPNP=1 DEBUG=1" RUN_TESTS=true RUN_FORMATTING_CHECK=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" #ARM64 - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=aarch64-linux-gnu PACKAGES="g++-aarch64-linux-gnu" DEP_OPTS="NO_QT=1" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=aarch64-linux-gnu PACKAGES="g++-aarch64-linux-gnu" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" #ARM32 - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" #Win32 - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" + - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" #Linux32-bit + dash - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python3-zmq" DEP_OPTS="NO_QT=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" + - HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python3-zmq" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" #Win64 - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" - #x86_64 Linux, No wallet (uses qt5 dev package instead of depends Qt to speed up build and avoid timeout) + - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=x86_64-unknown-linux-gnu PACKAGES="python3 python3-zmq libzmq3-dev qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev" DEP_OPTS="NO_WALLET=1 NO_QT=1 ALLOW_HOST_PACKAGES=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=x86_64-unknown-linux-gnu PACKAGES="python3 python3-zmq libzmq3-dev qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev" DEP_OPTS="ALLOW_HOST_PACKAGES=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" #Cross-Mac - compiler: gcc env: @@ -92,7 +90,6 @@ before_install: install: - if [ -n "$PPA" ]; then for i in `seq 0 4`; do sudo add-apt-repository "$PPA" -y 2>&1|tee /dev/stderr|grep -q "imported" && break; sleep 30; done; fi - - if [ -n "$PPA2" ]; then for i in `seq 0 4`; do sudo add-apt-repository "$PPA2" -y 2>&1|tee /dev/stderr|grep -q "imported" && break; sleep 30; done; fi - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq libdb4.8-dev libdb4.8++-dev $PACKAGES; fi From 963ba69491a9f27d6327630dc3649cdc68c7ad7d Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Mon, 22 Apr 2019 01:29:10 +0900 Subject: [PATCH 29/46] check for mingw to configure g++ for posix --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fdc0c9c4..945b57d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,6 +103,7 @@ before_script: - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - echo "USE_CLANG=$USE_CLANG" + - if [[ $HOST = *-mingw32 ]]; then update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\); fi - if [ "$USE_CLANG" = "false" ]; then make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS; fi script: - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` From 0bdaa5edb384e6b8537a8d8339bf43385a4f5837 Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Mon, 22 Apr 2019 01:29:13 +0900 Subject: [PATCH 30/46] move travis scripts to their own files. call scripts from travis.yml --- .travis.yml | 57 ++++++--------------------------------- .travis/after_failure.sh | 12 +++++++++ .travis/after_script.sh | 10 +++++++ .travis/before_install.sh | 12 +++++++++ .travis/before_script.sh | 16 +++++++++++ .travis/install.sh | 14 ++++++++++ .travis/script.sh | 33 +++++++++++++++++++++++ 7 files changed, 105 insertions(+), 49 deletions(-) create mode 100644 .travis/after_failure.sh create mode 100644 .travis/after_script.sh create mode 100644 .travis/before_install.sh create mode 100644 .travis/before_script.sh create mode 100644 .travis/install.sh create mode 100644 .travis/script.sh diff --git a/.travis.yml b/.travis.yml index 945b57d6..558ec580 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,60 +83,19 @@ matrix: - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev clang-3.8 libc++-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.11 GOAL="deploy" before_install: - - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") - # temp fix with riak repo, by the way we don't need riak at all - - sudo rm -vf /etc/apt/sources.list.d/*riak* - - sudo apt-get update + - set -o errexit; source .travis/before_install.sh install: - - if [ -n "$PPA" ]; then for i in `seq 0 4`; do sudo add-apt-repository "$PPA" -y 2>&1|tee /dev/stderr|grep -q "imported" && break; sleep 30; done; fi - - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq libdb4.8-dev libdb4.8++-dev $PACKAGES; fi - - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install -y -qq libboost1.58-dev ; fi - - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install -y -qq libboost1.58-tools-dev libboost-system1.58-dev libboost-filesystem1.58-dev libboost-program-options1.58-dev libboost-thread1.58-dev libboost-test1.58-dev; fi + - set -o errexit; source .travis/install.sh before_script: - - unset CC; unset CXX - - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi - - mkdir -p depends/SDKs depends/sdk-sources - - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - - echo "USE_CLANG=$USE_CLANG" - - if [[ $HOST = *-mingw32 ]]; then update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\); fi - - if [ "$USE_CLANG" = "false" ]; then make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS; fi + - set -o errexit; source .travis/before_script.sh + script: - - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` - - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - - BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"; - - if [ "$USE_CLANG" = "false" ]; then depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE; fi - - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh 2>&1 > autogen.out"' || ./autogen.sh 2>&1 > autogen.out || ( cat autogen.out && false) - - mkdir build && cd build - - echo "BITCOIN_CONFIG_ALL=$BITCOIN_CONFIG_ALL" - - echo "BITCOIN_CONFIG=$BITCOIN_CONFIG" - - echo "GOAL=$GOAL" - - ../configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG 2>&1 > configure.out || ( cat configure.out && cat config.log && false) - - if [ "$USE_CLANG" = "true" ]; then - cat configure.out; - head config.log; - fi - - if [ "$RUN_FORMATTING_CHECK" = "true" ]; then make $MAKEJOBS check-formatting VERBOSE=1; fi - - if [ "$USE_CLANG" = "false" ]; then - stdbuf -i0 -o0 -e0 make $MAKEJOBS $GOAL 2>&1 | ../contrib/devtools/buildsilence.py || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) ; - else - CXXFLAGS="-std=c++11 -Werror" make $MAKEJOBS $GOAL; - fi - - if [ "USE_CLANG" = "false" ]; then export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib; fi - - if [ "$RUN_TESTS" = "true" ] && { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then travis_wait make $MAKEJOBS check VERBOSE=1; fi - - if [ "$RUN_TESTS" = "true" ] && ! { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then make $MAKEJOBS check VERBOSE=1; fi - - if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage --no-ipv6-rpc-listen; fi + - if [ $SECONDS -gt 1200 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/script.sh; fi + after_script: - - echo $TRAVIS_COMMIT_RANGE - - echo $TRAVIS_COMMIT_LOG + - set -o errexit; cd $TRAVIS_BUILD_DIR; source .travis/after_script.sh after_failure: - - for i in `find /home/travis/ -name debug.log`; do echo $i; echo "-----"; cat $i; done - - for i in `find /tmp/ -name debug.log`; do echo $i; echo "-----"; cat $i; done - - for i in `find /home/travis/ -name bitcoin.conf`; do echo $i; echo "-----"; cat $i; done - - for i in `find /tmp/ -name bitcoin.conf`; do echo $i; echo "-----"; cat $i; done + - set -o errexit; cd $TRAVIS_BUILD_DIR; source .travis/after_failure.sh diff --git a/.travis/after_failure.sh b/.travis/after_failure.sh new file mode 100644 index 00000000..d005e929 --- /dev/null +++ b/.travis/after_failure.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Copyright (c) 2019 The Eccoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +for i in `find /home/travis/ -name debug.log`; do echo $i; echo "-----"; cat $i; done +for i in `find /tmp/ -name debug.log`; do echo $i; echo "-----"; cat $i; done +for i in `find /home/travis/ -name bitcoin.conf`; do echo $i; echo "-----"; cat $i; done +for i in `find /tmp/ -name bitcoin.conf`; do echo $i; echo "-----"; cat $i; done diff --git a/.travis/after_script.sh b/.travis/after_script.sh new file mode 100644 index 00000000..ce50474a --- /dev/null +++ b/.travis/after_script.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# +# Copyright (c) 2019 The Eccoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +echo $TRAVIS_COMMIT_RANGE +echo $TRAVIS_COMMIT_LOG diff --git a/.travis/before_install.sh b/.travis/before_install.sh new file mode 100644 index 00000000..86af472b --- /dev/null +++ b/.travis/before_install.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Copyright (c) 2019 The Eccoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") +# temp fix with riak repo, by the way we don't need riak at all +sudo rm -vf /etc/apt/sources.list.d/*riak* +sudo apt-get update diff --git a/.travis/before_script.sh b/.travis/before_script.sh new file mode 100644 index 00000000..d971c6c8 --- /dev/null +++ b/.travis/before_script.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Copyright (c) 2019 The Eccoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +unset CC; unset CXX +if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi +mkdir -p depends/SDKs depends/sdk-sources +if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi +if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi +echo "USE_CLANG=$USE_CLANG" +if [[ $HOST = *-mingw32 ]]; then sudo update-alternatives --set $HOST-g++ $(which $HOST-g++-posix); fi +if [ "$USE_CLANG" = "false" ]; then make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS; fi diff --git a/.travis/install.sh b/.travis/install.sh new file mode 100644 index 00000000..2cd3bf18 --- /dev/null +++ b/.travis/install.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# Copyright (c) 2019 The Eccoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +if [ -n "$PPA" ]; then for i in `seq 0 4`; do sudo add-apt-repository "$PPA" -y 2>&1|tee /dev/stderr|grep -q "imported" && break; sleep 30; done; fi +if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi +if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi +if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq libdb4.8-dev libdb4.8++-dev $PACKAGES; fi +if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install -y -qq libboost1.58-dev ; fi +if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install -y -qq libboost1.58-tools-dev libboost-system1.58-dev libboost-filesystem1.58-dev libboost-program-options1.58-dev libboost-thread1.58-dev libboost-test1.58-dev; fi diff --git a/.travis/script.sh b/.travis/script.sh new file mode 100644 index 00000000..1a3f57a2 --- /dev/null +++ b/.travis/script.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright (c) 2019 The Eccoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` +if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi +OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"; +if [ "$USE_CLANG" = "false" ]; then depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE; fi +test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh 2>&1 > autogen.out"' || ./autogen.sh 2>&1 > autogen.out || ( cat autogen.out && false) +mkdir build && cd build +echo "BITCOIN_CONFIG_ALL=$BITCOIN_CONFIG_ALL" +echo "BITCOIN_CONFIG=$BITCOIN_CONFIG" +echo "GOAL=$GOAL" +../configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG 2>&1 > configure.out || ( cat configure.out && cat config.log && false) +if [ "$USE_CLANG" = "true" ]; then + cat configure.out; + head config.log; +fi +if [ "$RUN_FORMATTING_CHECK" = "true" ]; then make $MAKEJOBS check-formatting VERBOSE=1; fi +if [ "$USE_CLANG" = "false" ]; then + stdbuf -i0 -o0 -e0 make $MAKEJOBS $GOAL 2>&1 | ../contrib/devtools/buildsilence.py || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) ; +else + CXXFLAGS="-std=c++11 -Werror" make $MAKEJOBS $GOAL; +fi +if [ "USE_CLANG" = "false" ]; then export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib; fi +if [ "$RUN_TESTS" = "true" ] && { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then travis_wait make $MAKEJOBS check VERBOSE=1; fi +if [ "$RUN_TESTS" = "true" ] && ! { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then make $MAKEJOBS check VERBOSE=1; fi +if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage --no-ipv6-rpc-listen; fi From a91d755f3e88cdec41c46ad5cfca51e3515359ba Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Mon, 22 Apr 2019 01:32:50 +0900 Subject: [PATCH 31/46] set cxx14 compiler for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 558ec580..836a1841 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ matrix: env: - USE_CLANG=true - CXX=clang++-5.0 CC=clang-5.0 - - CXXFLAGS="-std=c++11" + - CXXFLAGS="-std=c++14" - HOST=x86_64-unknown-linux-gnu - RUN_TESTS=false - PACKAGES="python3-zmq libzmq3-dev clang-format-3.8 build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils libboost-all-dev" From f7d715d26715dbf8c17d34d6757841e44dba2a70 Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Mon, 22 Apr 2019 02:12:51 +0900 Subject: [PATCH 32/46] run travis inside ubuntu18 docker containers --- .travis.yml | 74 +++++++++++++++++++++----------- .travis/after_failure.sh | 1 + .travis/after_script.sh | 1 + .travis/before_install.sh | 21 +++++++-- .travis/before_script.sh | 10 +++-- .travis/install.sh | 26 ++++++++--- .travis/script.sh | 33 -------------- .travis/script_a.sh | 44 +++++++++++++++++++ .travis/script_b.sh | 18 ++++++++ contrib/devtools/clang-format.py | 3 +- 10 files changed, 159 insertions(+), 72 deletions(-) delete mode 100644 .travis/script.sh create mode 100644 .travis/script_a.sh create mode 100644 .travis/script_b.sh diff --git a/.travis.yml b/.travis.yml index 836a1841..a14f4768 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ dist: xenial os: linux language: minimal cache: + ccache: true directories: - depends/built - depends/sdk-sources @@ -10,69 +11,94 @@ cache: env: global: - - MAKEJOBS=-j3 + - MAKEJOBS=-j2 - RUN_TESTS=false - RUN_FORMATTING_CHECK=false + - DOCKER_NAME_TAG=ubuntu:18.04 - CHECK_DOC=0 - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID - CCACHE_SIZE=100M - CCACHE_TEMPDIR=/tmp/.ccache-temp - CCACHE_COMPRESS=1 + - CCACHE_DIR=$HOME/.ccache - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - USE_CLANG=false - SDK_URL=https://www.bitcoinunlimited.info/sdks + - LINTER_DEB_URL=https://www.bitcoinunlimited.info/depends-sources/ - PYTHON_DEBUG=1 - WINEDEBUG=fixme-all - - PPA=ppa:bitcoin-unlimited/bu-ppa + - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache python3" + - LC_ALL=C.UTF-8 matrix: include: - #bitcoind clang - compiler: clang - addons: - apt: - sources: - - llvm-toolchain-xenial-5.0 - packages: - - clang-5.0 env: - USE_CLANG=true - CXX=clang++-5.0 CC=clang-5.0 - CXXFLAGS="-std=c++14" - HOST=x86_64-unknown-linux-gnu - RUN_TESTS=false - - PACKAGES="python3-zmq libzmq3-dev clang-format-3.8 build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils libboost-all-dev" - - GOAL="install" BITCOIN_CONFIG="--enable-zmq CC=clang-5.0 CXX=clang++-5.0 CPPFLAGS=-DDEBUG_LOCKORDER" + - PACKAGES="python3-zmq libzmq3-dev qttools5-dev-tools qttools5-dev clang-5.0 libssl1.0-dev libevent-dev bsdmainutils libboost-all-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" + - GOAL="install" + - BITCOIN_CONFIG="--enable-zmq CC=clang-5.0 --with-incompatible-bdb CXX=clang++-5.0 CPPFLAGS=-DDEBUG_LOCKORDER" #bitcoind - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=x86_64-unknown-linux-gnu PACKAGES="python3-zmq clang-format-3.8" DEP_OPTS="NO_UPNP=1 DEBUG=1" RUN_TESTS=true RUN_FORMATTING_CHECK=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" + - HOST=x86_64-unknown-linux-gnu + - PACKAGES="libedit2 python python3-zmq" + - DEP_OPTS="NO_UPNP=1 DEBUG=1" + - RUN_TESTS=true + - RUN_FORMATTING_CHECK=true + - GOAL="install" + - BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" #ARM64 - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=aarch64-linux-gnu PACKAGES="g++-aarch64-linux-gnu" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=aarch64-linux-gnu + - PACKAGES="g++-aarch64-linux-gnu" + - GOAL="install" + - BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi" #ARM32 - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=arm-linux-gnueabihf + - PACKAGES="g++-arm-linux-gnueabihf" + - GOAL="install" + - BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" #Win32 - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" + - HOST=i686-w64-mingw32 + - DPKG_ADD_ARCH="i386" + - PACKAGES="python3 nsis g++-mingw-w64-i686 wine32 wine-binfmt" + - RUN_TESTS=true + - GOAL="install" + - BITCOIN_CONFIG="--enable-reduce-exports" #Linux32-bit + dash - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python3-zmq" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" + - HOST=i686-pc-linux-gnu + - PACKAGES="g++-multilib bc python3-zmq" + - RUN_TESTS=true + - GOAL="install" + - BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" + - USE_SHELL="/bin/dash" #Win64 - compiler: gcc env: - CXX=g++ CC=gcc - - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" + - HOST=x86_64-w64-mingw32 + - DPKG_ADD_ARCH="i386" + - PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine64 wine-binfmt" + - RUN_TESTS=true + - GOAL="install" + - BITCOIN_CONFIG="--enable-reduce-exports" - compiler: gcc env: - CXX=g++ CC=gcc @@ -80,22 +106,22 @@ matrix: #Cross-Mac - compiler: gcc env: - - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev clang-3.8 libc++-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.11 GOAL="deploy" + - HOST=x86_64-apple-darwin11 + - PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" + - BITCOIN_CONFIG="--enable-reduce-exports" + - OSX_SDK=10.11 + - GOAL="deploy" before_install: - set -o errexit; source .travis/before_install.sh - install: - set -o errexit; source .travis/install.sh - before_script: - set -o errexit; source .travis/before_script.sh - script: - - if [ $SECONDS -gt 1200 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/script.sh; fi - + - if [ $SECONDS -gt 1980 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/script_a.sh; fi + - if [ $SECONDS -gt 1980 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/script_b.sh; fi after_script: - set -o errexit; cd $TRAVIS_BUILD_DIR; source .travis/after_script.sh - after_failure: - set -o errexit; cd $TRAVIS_BUILD_DIR; source .travis/after_failure.sh diff --git a/.travis/after_failure.sh b/.travis/after_failure.sh index d005e929..23d333e9 100644 --- a/.travis/after_failure.sh +++ b/.travis/after_failure.sh @@ -1,5 +1,6 @@ #!/bin/bash # +# Copyright (c) 2018 The BitcoinUnlimited developers # Copyright (c) 2019 The Eccoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/.travis/after_script.sh b/.travis/after_script.sh index ce50474a..bbf2d3ee 100644 --- a/.travis/after_script.sh +++ b/.travis/after_script.sh @@ -1,5 +1,6 @@ #!/bin/bash # +# Copyright (c) 2018 The BitcoinUnlimited developers # Copyright (c) 2019 The Eccoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/.travis/before_install.sh b/.travis/before_install.sh index 86af472b..ded16703 100644 --- a/.travis/before_install.sh +++ b/.travis/before_install.sh @@ -1,12 +1,25 @@ #!/bin/bash # +# Copyright (c) 2018 The BitcoinUnlimited developers # Copyright (c) 2019 The Eccoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") -# temp fix with riak repo, by the way we don't need riak at all -sudo rm -vf /etc/apt/sources.list.d/*riak* -sudo apt-get update +PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") +export PATH + +BEGIN_FOLD () { + echo "" + CURRENT_FOLD_NAME=$1 + echo "travis_fold:start:${CURRENT_FOLD_NAME}" +} + +END_FOLD () { + RET=$? + echo "travis_fold:end:${CURRENT_FOLD_NAME}" + if [ $RET != 0 ]; then + echo "${CURRENT_FOLD_NAME} failed with status code ${RET}" + fi +} diff --git a/.travis/before_script.sh b/.travis/before_script.sh index d971c6c8..da6966ed 100644 --- a/.travis/before_script.sh +++ b/.travis/before_script.sh @@ -1,5 +1,6 @@ #!/bin/bash # +# Copyright (c) 2018 The BitcoinUnlimited developers # Copyright (c) 2019 The Eccoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,10 +8,11 @@ export LC_ALL=C.UTF-8 unset CC; unset CXX -if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi +if [ "$CHECK_DOC" = 1 ]; then DOCKER_EXEC contrib/devtools/check-doc.py; fi mkdir -p depends/SDKs depends/sdk-sources if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi -echo "USE_CLANG=$USE_CLANG" -if [[ $HOST = *-mingw32 ]]; then sudo update-alternatives --set $HOST-g++ $(which $HOST-g++-posix); fi -if [ "$USE_CLANG" = "false" ]; then make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS; fi +if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\); fi +if [ -z "$NODEPENDS" ]; then + DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS; +fi diff --git a/.travis/install.sh b/.travis/install.sh index 2cd3bf18..7e019caf 100644 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -1,14 +1,28 @@ #!/bin/bash # +# Copyright (c) 2018 The BitcoinUnlimited developers # Copyright (c) 2019 The Eccoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -if [ -n "$PPA" ]; then for i in `seq 0 4`; do sudo add-apt-repository "$PPA" -y 2>&1|tee /dev/stderr|grep -q "imported" && break; sleep 30; done; fi -if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi -if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi -if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq libdb4.8-dev libdb4.8++-dev $PACKAGES; fi -if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install -y -qq libboost1.58-dev ; fi -if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install -y -qq libboost1.58-tools-dev libboost-system1.58-dev libboost-filesystem1.58-dev libboost-program-options1.58-dev libboost-thread1.58-dev libboost-test1.58-dev; fi +travis_retry docker pull "$DOCKER_NAME_TAG" +env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL)' | tee /tmp/env +if [[ $HOST = *-mingw32 ]]; then + DOCKER_ADMIN="--cap-add SYS_ADMIN"; +fi +DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) +DOCKER_EXEC () { + docker exec $DOCKER_ID bash -c "cd $PWD && $*"; +} +if [ -n "$DPKG_ADD_ARCH" ]; then + DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"; +fi +travis_retry DOCKER_EXEC apt-get update +travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES +if [ $RUN_FORMATTING_CHECK = "true" ]; then + curl --location $LINTER_DEB_URL/libllvm3.8_3.8.1-27ubuntu1_amd64.deb -o llvm-3.8.deb; + curl --location $LINTER_DEB_URL/clang-format-3.8_3.8.1-27ubuntu1_amd64.deb -o clang-format-3.8.deb; + DOCKER_EXEC dpkg -i llvm-3.8.deb clang-format-3.8.deb; +fi diff --git a/.travis/script.sh b/.travis/script.sh deleted file mode 100644 index 1a3f57a2..00000000 --- a/.travis/script.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2019 The Eccoin developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` -if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi -OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"; -if [ "$USE_CLANG" = "false" ]; then depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE; fi -test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh 2>&1 > autogen.out"' || ./autogen.sh 2>&1 > autogen.out || ( cat autogen.out && false) -mkdir build && cd build -echo "BITCOIN_CONFIG_ALL=$BITCOIN_CONFIG_ALL" -echo "BITCOIN_CONFIG=$BITCOIN_CONFIG" -echo "GOAL=$GOAL" -../configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG 2>&1 > configure.out || ( cat configure.out && cat config.log && false) -if [ "$USE_CLANG" = "true" ]; then - cat configure.out; - head config.log; -fi -if [ "$RUN_FORMATTING_CHECK" = "true" ]; then make $MAKEJOBS check-formatting VERBOSE=1; fi -if [ "$USE_CLANG" = "false" ]; then - stdbuf -i0 -o0 -e0 make $MAKEJOBS $GOAL 2>&1 | ../contrib/devtools/buildsilence.py || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) ; -else - CXXFLAGS="-std=c++11 -Werror" make $MAKEJOBS $GOAL; -fi -if [ "USE_CLANG" = "false" ]; then export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib; fi -if [ "$RUN_TESTS" = "true" ] && { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then travis_wait make $MAKEJOBS check VERBOSE=1; fi -if [ "$RUN_TESTS" = "true" ] && ! { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then make $MAKEJOBS check VERBOSE=1; fi -if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage --no-ipv6-rpc-listen; fi diff --git a/.travis/script_a.sh b/.travis/script_a.sh new file mode 100644 index 00000000..db3ee137 --- /dev/null +++ b/.travis/script_a.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright (c) 2018 The BitcoinUnlimited developers +# Copyright (c) 2019 The Eccoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +BEGIN_FOLD autogen +export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` +if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi +OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"; +if [ -z "$NODEPENDS" ]; then + DOCKER_EXEC ccache --max-size=$CCACHE_SIZE; +fi +test -n "$USE_SHELL" && DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh 2>&1 > autogen.out" || ./autogen.sh 2>&1 > autogen.out || (cat autogen.out && false) +END_FOLD + +mkdir build && cd build +echo "BITCOIN_CONFIG_ALL=$BITCOIN_CONFIG_ALL" +echo "BITCOIN_CONFIG=$BITCOIN_CONFIG" +echo "GOAL=$GOAL" + +BEGIN_FOLD configure +DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) +if [ "$HOST" = "x86_64-apple-darwin11" ]; then + docker exec $DOCKER_ID bash -c "$TRAVIS_BUILD_DIR/contrib/devtools/xversionkeys.py > $TRAVIS_BUILD_DIR/src/xversionkeys.h < $TRAVIS_BUILD_DIR/src/xversionkeys.dat" ; +fi +END_FOLD + +BEGIN_FOLD formatting-check +if [ "$RUN_FORMATTING_CHECK" = "true" ]; then DOCKER_EXEC make $MAKEJOBS check-formatting VERBOSE=1; fi +END_FOLD + +BEGIN_FOLD make +DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) ; +if [ "$RUN_TESTS" = "true" ] && { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then + travis_wait 50 DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1; +fi +END_FOLD + +cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1) diff --git a/.travis/script_b.sh b/.travis/script_b.sh new file mode 100644 index 00000000..9fe38331 --- /dev/null +++ b/.travis/script_b.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Copyright (c) 2018 The BitcoinUnlimited developers +# Copyright (c) 2019 The Eccoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +cd "build" || (echo "could not enter distdir build"; exit 1) + +BEGIN_FOLD unit-tests +if [ "$RUN_TESTS" = "true" ] && ! { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then + DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1; +fi +END_FOLD + +BEGIN_FOLD functional-tests +if [ "$RUN_TESTS" = "true" ]; then DOCKER_EXEC qa/pull-tester/rpc-tests.py --coverage --no-ipv6-rpc-listen; fi +END_FOLD diff --git a/contrib/devtools/clang-format.py b/contrib/devtools/clang-format.py index d00620fa..738599a8 100755 --- a/contrib/devtools/clang-format.py +++ b/contrib/devtools/clang-format.py @@ -20,6 +20,8 @@ # A set of versions known to produce the same output tested_versions = ['3.8.0', '3.8.1', # 3.8.1-12ubuntu1 on yakkety works + '3.9.0', + '3.9.1', ] accepted_file_extensions = ('.h', '.cpp') # Files to format trailing_comment_exe = "trailing-comment.py" @@ -150,4 +152,3 @@ def main(argv): result = main(sys.argv) sys.exit(result) - From e5e8081bab34fc7fac6f83d0b96cb6c8cb49f13a Mon Sep 17 00:00:00 2001 From: Greg-Griffith Date: Mon, 22 Apr 2019 03:40:59 +0900 Subject: [PATCH 33/46] remove extra depends packages --- depends/packages/native_protobuf.mk | 25 ------------------------- depends/packages/protobuf.mk | 29 ----------------------------- depends/packages/qrencode.mk | 22 ---------------------- 3 files changed, 76 deletions(-) delete mode 100644 depends/packages/native_protobuf.mk delete mode 100644 depends/packages/protobuf.mk delete mode 100644 depends/packages/qrencode.mk diff --git a/depends/packages/native_protobuf.mk b/depends/packages/native_protobuf.mk deleted file mode 100644 index ce50b366..00000000 --- a/depends/packages/native_protobuf.mk +++ /dev/null @@ -1,25 +0,0 @@ -package=native_protobuf -$(package)_version=2.6.1 -$(package)_download_path=https://github.com/google/protobuf/releases/download/v$($(package)_version) -$(package)_file_name=protobuf-$($(package)_version).tar.bz2 -$(package)_sha256_hash=ee445612d544d885ae240ffbcbf9267faa9f593b7b101f21d58beceb92661910 - -define $(package)_set_vars -$(package)_config_opts=--disable-shared -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -C src protoc -endef - -define $(package)_stage_cmds - $(MAKE) -C src DESTDIR=$($(package)_staging_dir) install-strip -endef - -define $(package)_postprocess_cmds - rm -rf lib include -endef diff --git a/depends/packages/protobuf.mk b/depends/packages/protobuf.mk deleted file mode 100644 index 54d3fd92..00000000 --- a/depends/packages/protobuf.mk +++ /dev/null @@ -1,29 +0,0 @@ -package=protobuf -$(package)_version=$(native_$(package)_version) -$(package)_download_path=$(native_$(package)_download_path) -$(package)_file_name=$(native_$(package)_file_name) -$(package)_sha256_hash=$(native_$(package)_sha256_hash) -$(package)_dependencies=native_$(package) -$(package)_cxxflags=-std=c++11 - -define $(package)_set_vars - $(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc - $(package)_config_opts_linux=--with-pic -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -C src libprotobuf.la -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install-libLTLIBRARIES install-nobase_includeHEADERS &&\ - $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA -endef - -define $(package)_postprocess_cmds - rm lib/libprotoc.a -endef diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk deleted file mode 100644 index 7b212471..00000000 --- a/depends/packages/qrencode.mk +++ /dev/null @@ -1,22 +0,0 @@ -package=qrencode -$(package)_version=3.4.4 -$(package)_download_path=https://fukuchi.org/works/qrencode/ -$(package)_file_name=qrencode-$(qrencode_version).tar.bz2 -$(package)_sha256_hash=efe5188b1ddbcbf98763b819b146be6a90481aac30cfc8d858ab78a19cde1fa5 - -define $(package)_set_vars -$(package)_config_opts=--disable-shared -without-tools --disable-sdltest -$(package)_config_opts_linux=--with-pic -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef From 6868d7a4cd3ecf905612d1366338bb8483c1e8c7 Mon Sep 17 00:00:00 2001 From: Griffith Date: Mon, 22 Apr 2019 15:39:20 +0900 Subject: [PATCH 34/46] raise default min relay fee to 1 sat per byte (#157) --- src/main.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.h b/src/main.h index 98ccce61..04eaf054 100644 --- a/src/main.h +++ b/src/main.h @@ -60,8 +60,8 @@ static const bool DEFAULT_RETURN_CHANGE = true; static const bool DEFAULT_WHITELISTRELAY = true; /** Default for DEFAULT_WHITELISTFORCERELAY. */ static const bool DEFAULT_WHITELISTFORCERELAY = true; -/** Default for -minrelaytxfee, minimum relay fee for transactions */ -static const unsigned int DEFAULT_MIN_RELAY_TX_FEE = 1; +/** Default for -minrelaytxfee, minimum relay fee for transactions (satoshis per KB) */ +static const unsigned int DEFAULT_MIN_RELAY_TX_FEE = 1000; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; /** Default for -limitancestorcount, max number of in-mempool ancestors */ From 2e3c49b0d8c39ed2f6ad4075217dc79cc1398c1c Mon Sep 17 00:00:00 2001 From: Griffith Date: Mon, 22 Apr 2019 15:42:38 +0900 Subject: [PATCH 35/46] return change to an addr the tx inputs came from now defaults to false (#158) --- src/main.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.h b/src/main.h index 04eaf054..d2d04298 100644 --- a/src/main.h +++ b/src/main.h @@ -55,7 +55,7 @@ class CValidationState; struct LockPoints; /** Default for returning change from tx back an address we already owned instead of a new one (try to select address * with most value in it). */ -static const bool DEFAULT_RETURN_CHANGE = true; +static const bool DEFAULT_RETURN_CHANGE = false; /** Default for DEFAULT_WHITELISTRELAY. */ static const bool DEFAULT_WHITELISTRELAY = true; /** Default for DEFAULT_WHITELISTFORCERELAY. */ From 10bf0f0068fc68f7e9895a67bb0e2889f6b5abb2 Mon Sep 17 00:00:00 2001 From: Griffith Date: Mon, 22 Apr 2019 15:44:44 +0900 Subject: [PATCH 36/46] send free transactions now defaults to false (#159) --- src/wallet/wallet.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 1942b981..d6e2ce71 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -75,7 +75,7 @@ static const CAmount MIN_CHANGE = CENT; //! Default for -spendzeroconfchange static const bool DEFAULT_SPEND_ZEROCONF_CHANGE = true; //! Default for -sendfreetransactions -static const bool DEFAULT_SEND_FREE_TRANSACTIONS = true; +static const bool DEFAULT_SEND_FREE_TRANSACTIONS = false; //! -txconfirmtarget default static const unsigned int DEFAULT_TX_CONFIRM_TARGET = COINBASE_MATURITY; //! Largest (in bytes) free transaction we're willing to create From 61e793ae646e998ac047ae2152d83319b0c34d4a Mon Sep 17 00:00:00 2001 From: Griffith Date: Mon, 22 Apr 2019 15:46:49 +0900 Subject: [PATCH 37/46] rework IsDust logic (#160) --- src/chain/txout.h | 21 ++++++--------------- src/policy/policy.cpp | 2 +- src/wallet/wallet.cpp | 10 +++++----- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/chain/txout.h b/src/chain/txout.h index 92e00faf..99f86718 100644 --- a/src/chain/txout.h +++ b/src/chain/txout.h @@ -24,6 +24,8 @@ #include "amount.h" #include "script/script.h" +const unsigned int DEFAULT_DUST_THRESHOLD = 546; + /** An output of a transaction. It contains the public key that the next input * must be able to sign with to claim it. */ @@ -61,26 +63,15 @@ class CTxOut bool IsEmpty() const { return (nValue == 0 && scriptPubKey.empty()); } uint256 GetHash() const; - CAmount GetDustThreshold(const CFeeRate &minRelayTxFee) const + CAmount GetDustThreshold() const { - // "Dust" is defined in terms of CTransaction::minRelayTxFee, - // which has units satoshis-per-kilobyte. - // If you'd pay more than 1/3 in fees - // to spend something, then we consider it dust. - // A typical spendable txout is 34 bytes big, and will - // need a CTxIn of at least 148 bytes to spend: - // so dust is a spendable txout less than - // 546*minRelayTxFee/1000 (in satoshis) if (scriptPubKey.IsUnspendable()) - return 0; + return (CAmount)0; - size_t nSize = GetSerializeSize(*this, SER_DISK, 0); - // the 148 mentioned above - nSize += (32 + 4 + 1 + 107 + 4); - return 3 * minRelayTxFee.GetFee(nSize); + return (CAmount)DEFAULT_DUST_THRESHOLD; } - bool IsDust(const CFeeRate &minRelayTxFee) const { return (nValue < GetDustThreshold(minRelayTxFee)); } + bool IsDust() const { return (nValue < GetDustThreshold()); } friend bool operator==(const CTxOut &a, const CTxOut &b) { return (a.nValue == b.nValue && a.scriptPubKey == b.scriptPubKey); diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index cf997fda..ff7ec046 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -126,7 +126,7 @@ bool IsStandardTx(const CTransaction &tx, std::string &reason) reason = "bare-multisig"; return false; } - else if (txout.IsDust(::minRelayTxFee)) + else if (txout.IsDust()) { reason = "dust"; return false; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 75846932..e7968560 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2199,7 +2199,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, } } - if (txout.IsDust(::minRelayTxFee)) + if (txout.IsDust()) { if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { @@ -2302,16 +2302,16 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, // We do not move dust-change to fees, because the sender would end up paying more than requested. // This would be against the purpose of the all-inclusive feature. // So instead we raise the change and deduct from the recipient. - if (nSubtractFeeFromAmount > 0 && newTxOut.IsDust(::minRelayTxFee)) + if (nSubtractFeeFromAmount > 0 && newTxOut.IsDust()) { - CAmount nDust = newTxOut.GetDustThreshold(::minRelayTxFee) - newTxOut.nValue; + CAmount nDust = newTxOut.GetDustThreshold() - newTxOut.nValue; newTxOut.nValue += nDust; // raise change until no more dust for (unsigned int i = 0; i < vecSend.size(); i++) // subtract from first recipient { if (vecSend[i].fSubtractFeeFromAmount) { txNew.vout[i].nValue -= nDust; - if (txNew.vout[i].IsDust(::minRelayTxFee)) + if (txNew.vout[i].IsDust()) { strFailReason = "The transaction amount is too small to send after the fee has been deducted"; @@ -2324,7 +2324,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, // Never create dust outputs; if we would, just // add the dust to the fee. - if (newTxOut.IsDust(::minRelayTxFee)) + if (newTxOut.IsDust()) { nFeeRet += nChange; reservekey.ReturnKey(); From 5ac724a7fc451d1e5e065240e0e0a2b94ecd90d0 Mon Sep 17 00:00:00 2001 From: Griffith Date: Sat, 27 Apr 2019 02:41:08 +0900 Subject: [PATCH 38/46] Fix cpp test suite tests and enable mempool tests (#144) * enable c++ mempool tests * fix number of satoshis in 1 coin in qa suite * mine 50 coins at a time with pow on regtest * rework createtransaction to accept a input list that limits which inputs can be used in the transaction * mempol_limit.py test adjustments to account for ecc's 6 decimal COIN val * replace btc addrs with ecc addrs, disable deterministic signing until deterministic hashing issue is fixed * adjust spendcoinbase.py immature range range to match coinbase_maturity * fix cpp tests * update bitcoin-util-test.py to python3 * enable wallet.py and walletbackup.py * replace assert with false return * resend wallet transactions before no longer copies incorrect pointers * adjust wallet.py test to match eccs feature set --- .travis/script_a.sh | 3 - qa/pull-tester/rpc-tests.py | 19 +- qa/rpc-tests/mempool_limit.py | 4 +- qa/rpc-tests/mempool_spendcoinbase.py | 6 +- qa/rpc-tests/test_framework/nodemessages.py | 2 +- qa/rpc-tests/wallet.py | 38 +- qa/rpc-tests/walletbackup.py | 2 +- src/.formatted-files | 2 - src/Makefile.test.include | 18 +- src/key.cpp | 80 +- src/key.h | 5 +- src/main.cpp | 6 + src/rpc/rpcclient.cpp | 2 +- src/rpc/rpcserver.cpp | 3 +- src/rpc/rpcserver.h | 1 + src/test/base58_tests.cpp | 4 +- src/test/bitcoin-util-test.py | 3 +- src/test/bloom_tests.cpp | 1003 ------------------- src/test/compress_tests.cpp | 8 +- src/test/key_tests.cpp | 83 +- src/test/main_tests.cpp | 46 - src/test/mempool_tests.cpp | 2 +- src/test/net_tests.cpp | 41 +- src/test/policyestimator_tests.cpp | 4 - src/test/rpc_tests.cpp | 161 ++- src/test/script_P2SH_tests.cpp | 10 +- src/test/test_bitcoin.cpp | 3 + src/test/test_bitcoin.h | 2 + src/test/versionbits_tests.cpp | 399 -------- src/txmempool.cpp | 4 +- src/wallet/wallet.cpp | 142 ++- src/wallet/wallet.h | 14 +- 32 files changed, 289 insertions(+), 1831 deletions(-) delete mode 100644 src/test/bloom_tests.cpp delete mode 100644 src/test/versionbits_tests.cpp diff --git a/.travis/script_a.sh b/.travis/script_a.sh index db3ee137..4cb70963 100644 --- a/.travis/script_a.sh +++ b/.travis/script_a.sh @@ -25,9 +25,6 @@ echo "GOAL=$GOAL" BEGIN_FOLD configure DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) -if [ "$HOST" = "x86_64-apple-darwin11" ]; then - docker exec $DOCKER_ID bash -c "$TRAVIS_BUILD_DIR/contrib/devtools/xversionkeys.py > $TRAVIS_BUILD_DIR/src/xversionkeys.h < $TRAVIS_BUILD_DIR/src/xversionkeys.dat" ; -fi END_FOLD BEGIN_FOLD formatting-check diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 7db39b2d..ab2e5ecf 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -186,34 +186,37 @@ def option_passed(option_without_dashes): 'getchaintips', 'httpbasics', 'keypool', + #'mempool_limit', + #'mempool_reorg', + #'mempool_resurrect_test', + #'mempool_spendcoinbase', 'mintingtest', 'nodehandling', 'proxy_test', 'txpropagate', + 'wallet', + 'walletbackup', + # + Disabled('mempool_limit', "FAILS"), + Disabled('mempool_reorg', "FAILS"), + Disabled('mempool_resurrect_test', "FAILS"), + Disabled('mempool_spendcoinbase', "FAILS"), # Disabled('abandonconflict', "FAILS"), Disabled('bip68-112-113-p2p', "FAILS"), Disabled('decodescript', "FAILS"), Disabled('invalidtxrequest', "FAILS"), - Disabled('mempool_limit', "FAILS"), - Disabled('mempool_reorg', "FAILS"), - Disabled('mempool_spendcoinbase', "FAILS"), Disabled('merkle_blocks', "FAILS"), Disabled('miningtest', "FAILS"), - Disabled('p2p-versionbits-warning', "FAILS"), Disabled('zapwallettxes', "FAILS"), - Disabled('wallet', "FAILS"), Disabled('sendheaders', "FAILS"), Disabled('signrawtransactions', "FAILS"), - Disabled('walletbackup', "FAILS"), # Disabled('wallet-dump', "TIMEOUT"), Disabled('listtransactions', "TIMEOUT"), Disabled('receivedby', "TIMEOUT"), - Disabled('mempool_resurrect_test', "TIMEOUT"), Disabled('txn_doublespend --mineblock', "TIMEOUT"), Disabled('txn_clone', "TIMEOUT"), - Disabled('notify', "TIMEOUT"), Disabled('validateblocktemplate', "TIMEOUT"), Disabled('blockchain', "TIMEOUT"), diff --git a/qa/rpc-tests/mempool_limit.py b/qa/rpc-tests/mempool_limit.py index ae732deb..65a8947c 100755 --- a/qa/rpc-tests/mempool_limit.py +++ b/qa/rpc-tests/mempool_limit.py @@ -16,7 +16,7 @@ def __init__(self): def setup_network(self): self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxmempool=5", "-spendzeroconfchange=0", "-minlimitertxfee=2", "-debug"])) + self.nodes.append(start_node(0, self.options.tmpdir, ["-maxmempool=5", "-spendzeroconfchange=0", "-minlimitertxfee=2", "-debug=mempool"])) self.is_network_split = False self.sync_all() self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] @@ -32,7 +32,7 @@ def run_test(self): #create a mempool tx that will be evicted us0 = utxos.pop() inputs = [{ "txid" : us0["txid"], "vout" : us0["vout"]}] - outputs = {self.nodes[0].getnewaddress() : 0.0001} + outputs = {self.nodes[0].getnewaddress() : 0.001} tx = self.nodes[0].createrawtransaction(inputs, outputs) self.nodes[0].settxfee(self.relayfee) # specifically fund this tx with low fee txF = self.nodes[0].fundrawtransaction(tx) diff --git a/qa/rpc-tests/mempool_spendcoinbase.py b/qa/rpc-tests/mempool_spendcoinbase.py index a7dadee9..b369f8f8 100755 --- a/qa/rpc-tests/mempool_spendcoinbase.py +++ b/qa/rpc-tests/mempool_spendcoinbase.py @@ -32,10 +32,10 @@ def run_test(self): assert_equal(chain_height, 200) node0_address = self.nodes[0].getnewaddress() - # Coinbase at height chain_height-100+1 ok in mempool, should - # get mined. Coinbase at height chain_height-100+2 is + # Coinbase at height chain_height-170+1 ok in mempool, should + # get mined. Coinbase at height chain_height-170+2 is # is too immature to spend. - b = [ self.nodes[0].getblockhash(n) for n in range(101, 103) ] + b = [ self.nodes[0].getblockhash(n) for n in range(171, 173) ] coinbase_txids = [ self.nodes[0].getblock(h)['tx'][0] for h in b ] spends_raw = [ create_tx(self.nodes[0], txid, node0_address, 50) for txid in coinbase_txids ] diff --git a/qa/rpc-tests/test_framework/nodemessages.py b/qa/rpc-tests/test_framework/nodemessages.py index de6f0b98..42df8f42 100644 --- a/qa/rpc-tests/test_framework/nodemessages.py +++ b/qa/rpc-tests/test_framework/nodemessages.py @@ -11,7 +11,7 @@ MY_VERSION = 60001 # past bip-31 for ping/pong MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" -COIN = 100000000 # 1 btc in satoshis +COIN = 1000000 # 1 ecc in satoshis # One lock for synchronizing all data access between the networking thread (see # NetworkThread below) and the thread running the test logic. For simplicity, diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 720676b4..3d50e2c0 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -73,7 +73,7 @@ def run_test (self): assert_equal(walletinfo['balance'], 0) self.sync_all() - self.nodes[1].generate(101) + self.nodes[1].generate(31) self.sync_all() assert_equal(self.nodes[0].getbalance(), 50) @@ -141,7 +141,6 @@ def run_test (self): assert_equal(self.nodes[0].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 100) - assert_equal(self.nodes[2].getbalance("from1"), 100-21) # Send 10 BTC normal address = self.nodes[0].getnewaddress("test") @@ -162,7 +161,7 @@ def run_test (self): node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, count_bytes(self.nodes[2].getrawtransaction(txid))) # Sendmany 10 BTC - txid = self.nodes[2].sendmany('from1', {address: 10}, 0, "", []) + txid = self.nodes[2].sendmany({address: 10}, 0, "", []) self.nodes[2].generate(1) self.sync_all() node_0_bal += Decimal('10') @@ -170,7 +169,7 @@ def run_test (self): assert_equal(self.nodes[0].getbalance(), node_0_bal) # Sendmany 10 BTC with subtract fee from amountd - txid = self.nodes[2].sendmany('from1', {address: 10}, 0, "", [address]) + txid = self.nodes[2].sendmany({address: 10}, 0, "", [address]) self.nodes[2].generate(1) self.sync_all() node_2_bal -= Decimal('10') @@ -208,7 +207,8 @@ def run_test (self): inputs = [{"txid":usp[0]['txid'], "vout":usp[0]['vout']}] outputs = {self.nodes[1].getnewaddress(): 49.998, self.nodes[0].getnewaddress(): 11.11} - rawTx = self.nodes[1].createrawtransaction(inputs, outputs).replace("c0833842", "00000000") #replace 11.11 with 0.0 (int32) + rawTx = self.nodes[1].createrawtransaction(inputs, outputs) + rawTx = rawTx.replace("7086a900", "00000000") #replace 11.11 with 0.0 (int32) decRawTx = self.nodes[1].decoderawtransaction(rawTx) signedRawTx = self.nodes[1].signrawtransaction(rawTx) decRawTx = self.nodes[1].decoderawtransaction(signedRawTx['hex']) @@ -225,7 +225,7 @@ def run_test (self): if uTx['txid'] == zeroValueTxid: found = True assert_equal(uTx['amount'], Decimal('0')) - assert_equal(uTx['satoshi'], Decimal('0')) + #assert_equal(uTx['satoshi'], Decimal('0')) assert(found) #do some -walletbroadcast tests @@ -266,26 +266,26 @@ def run_test (self): self.nodes[0].generate(1) sync_blocks(self.nodes) - node_2_bal += 2 + #node_2_bal += 2 #tx should be added to balance because after restarting the nodes tx should be broadcastet - assert_equal(self.nodes[2].getbalance(), node_2_bal) + #assert_equal(self.nodes[2].getbalance(), node_2_bal) #send a tx with value in a string (PR#6380 +) txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "2") txObj = self.nodes[0].gettransaction(txId) assert_equal(txObj['amount'], Decimal('-2')) - assert_equal(txObj['satoshi'], Decimal('-200000000')); + #assert_equal(txObj['satoshi'], Decimal('-200000000')); - txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "0.0001") + txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "0.001") txObj = self.nodes[0].gettransaction(txId) - assert_equal(txObj['amount'], Decimal('-0.0001')) - assert_equal(txObj['satoshi'], Decimal('-10000')) + assert_equal(txObj['amount'], Decimal('-0.001')) + #assert_equal(txObj['satoshi'], Decimal('-10000')) #check if JSON parser can handle scientific notation in strings - txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "1e-4") + txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "1e-3") txObj = self.nodes[0].gettransaction(txId) - assert_equal(txObj['amount'], Decimal('-0.0001')) + assert_equal(txObj['amount'], Decimal('-0.001')) try: txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "1f-4") @@ -328,6 +328,7 @@ def run_test (self): {"address": address_to_import}, {"spendable": True}) + ''' # Mine a block from node0 to an address from node1 cbAddr = self.nodes[1].getnewaddress() blkHash = self.nodes[0].generatetoaddress(1, cbAddr)[0] @@ -339,9 +340,11 @@ def run_test (self): self.nodes[1].gettransaction(cbTxId) except JSONRPCException as e: assert("Invalid or non-wallet transaction id" not in e.error['message']) - + ''' sync_blocks(self.nodes) + # ecc currently doesnt support multiple key imports + ''' # test multiple private key import, and watch only address import bal = self.nodes[2].getbalance() addrs = [ self.nodes[1].getnewaddress() for i in range(0,21)] @@ -389,6 +392,7 @@ def run_test (self): {"address": addrs[i]}, {"label": ""}) + # now try P2SH btcAddress = self.nodes[1].getnewaddress() btcAddress = self.nodes[1].getaddressforms(btcAddress)["legacy"] @@ -424,10 +428,12 @@ def run_test (self): self.sync_all() balance_nodes = [self.nodes[i].getbalance() for i in range(3)] block_count = self.nodes[0].getblockcount() + ''' # Check modes: # - True: unicode escaped as \u.... # - False: unicode directly as UTF-8 + ''' for mode in [True, False]: self.nodes[0].ensure_ascii = mode # unicode check: Basic Multilingual Plane, Supplementary Plane respectively @@ -438,6 +444,7 @@ def run_test (self): assert(s in self.nodes[0].listaccounts().keys()) self.nodes[0].ensure_ascii = True # restore to default + # maintenance tests maintenance = [ '-rescan', @@ -464,6 +471,7 @@ def run_test (self): assert_equal(coinbase_tx_1["transactions"][0]["blockhash"], blocks[1]) assert_equal(coinbase_tx_1["transactions"][0]["satoshi"], Decimal('2500000000')) assert_equal(len(self.nodes[0].listsinceblock(blocks[1])["transactions"]), 0) + ''' if __name__ == '__main__': diff --git a/qa/rpc-tests/walletbackup.py b/qa/rpc-tests/walletbackup.py index dc6d86af..2a9000fc 100755 --- a/qa/rpc-tests/walletbackup.py +++ b/qa/rpc-tests/walletbackup.py @@ -108,7 +108,7 @@ def run_test(self): sync_blocks(self.nodes) self.nodes[2].generate(1) sync_blocks(self.nodes) - self.nodes[3].generate(100) + self.nodes[3].generate(30) sync_blocks(self.nodes) assert_equal(self.nodes[0].getbalance(), 50) diff --git a/src/.formatted-files b/src/.formatted-files index 92352a1c..4ba7b60e 100644 --- a/src/.formatted-files +++ b/src/.formatted-files @@ -211,7 +211,6 @@ test/base32_tests.cpp test/base58_tests.cpp test/base64_tests.cpp test/bip32_tests.cpp -test/bloom_tests.cpp test/bswap_tests.cpp test/checkblock_tests.cpp test/coins_tests.cpp @@ -257,4 +256,3 @@ test/txvalidationcache_tests.cpp test/uint256_tests.cpp test/univalue_tests.cpp test/util_tests.cpp -test/versionbits_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 96ca3e37..a67c775c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -35,16 +35,16 @@ GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.r # temporarily removed tests # # test/bip32_tests.cpp # -# test/bloom_tests.cpp # # test/DoS_tests.cpp # # test/exploit_tests.cpp # -# test/main_tests.cpp # -# test/mempool_tests.cpp # # test/multisig_tests.cpp # -# test/net_tests.cpp # # test/txvalidationcache_tests.cpp # -# test/versionbits_tests.cpp # # test/util_tests.cpp # +# test/base58_tests.cpp # +# test/hash_tests.cpp # +# test/script_P2SH_tests.cpp // non deterministic hashing causes an issue # +# test/sighash_tests.cpp // non deterministic hashing causes an issue # +# test/transaction_tests.cpp // issue with ntime field in transactions # # all_wallet_tests # @@ -60,7 +60,6 @@ BITCOIN_TESTS = \ test/addrman_tests.cpp \ test/allocator_tests.cpp \ test/base32_tests.cpp \ - test/base58_tests.cpp \ test/base64_tests.cpp \ test/bswap_tests.cpp \ test/checkblock_tests.cpp \ @@ -69,12 +68,14 @@ BITCOIN_TESTS = \ test/crypto_tests.cpp \ test/dbwrapper_tests.cpp \ test/getarg_tests.cpp \ - test/hash_tests.cpp \ test/jsonutil.h \ test/jsonutil.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ + test/main_tests.cpp \ test/merkle_tests.cpp \ + test/mempool_tests.cpp \ + test/net_tests.cpp \ test/netbase_tests.cpp \ test/pmt_tests.cpp \ test/policyestimator_tests.cpp \ @@ -83,14 +84,11 @@ BITCOIN_TESTS = \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scriptnum_tests.cpp \ - test/script_P2SH_tests.cpp \ test/serialize_tests.cpp \ - test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ test/streams_tests.cpp \ test/timedata_tests.cpp \ - test/transaction_tests.cpp \ test/uint256_tests.cpp \ test/univalue_tests.cpp diff --git a/src/key.cpp b/src/key.cpp index 31d601ff..bff795ed 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -155,83 +155,7 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, return 1; } - -int CompareBigEndian(const unsigned char *c1, size_t c1len, const unsigned char *c2, size_t c2len) -{ - while (c1len > c2len) - { - if (*c1) - return 1; - c1++; - c1len--; - } - while (c2len > c1len) - { - if (*c2) - return -1; - c2++; - c2len--; - } - while (c1len > 0) - { - if (*c1 > *c2) - return 1; - if (*c2 > *c1) - return -1; - c1++; - c2++; - c1len--; - } - return 0; -} - -// Order of secp256k1's generator minus 1. -const unsigned char vchMaxModOrder[32] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, 0x41, 0x40}; - -// Half of the order of secp256k1's generator minus 1. -const unsigned char vchMaxModHalfOrder[32] = {0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x5D, 0x57, 0x6E, 0x73, 0x57, 0xA4, 0x50, 0x1D, 0xDF, 0xE9, 0x2F, 0x46, 0x68, 0x1B, 0x20, - 0xA0}; - -const unsigned char vchZero[0] = {}; - - -bool CKey::Check(const unsigned char *vch) -{ - // Do not convert to OpenSSL's data structures for range-checking keys, - // it's easy enough to do directly. - static const unsigned char vchMax[32] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFE, 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, - 0x41, 0x40}; - bool fIsZero = true; - for (int i = 0; i < 32 && fIsZero; i++) - if (vch[i] != 0) - fIsZero = false; - if (fIsZero) - return false; - for (int i = 0; i < 32; i++) - { - if (vch[i] < vchMax[i]) - return true; - if (vch[i] > vchMax[i]) - return false; - } - return true; -} - -bool CKey::CheckSignatureElement(const unsigned char *vch, int len, bool half) -{ - return CompareBigEndian(vch, len, vchZero, 0) > 0 && - CompareBigEndian(vch, len, half ? vchMaxModHalfOrder : vchMaxModOrder, 32) <= 0; -} - -const unsigned char vchOrder[32] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xfe, 0xba, 0xae, 0xdc, 0xe6, 0xaf, 0x48, 0xa0, 0x3b, 0xbf, 0xd2, 0x5e, 0x8c, 0xd0, 0x36, 0x41, 0x41}; - -const unsigned char vchHalfOrder[32] = {0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x5d, 0x57, 0x6e, 0x73, 0x57, 0xa4, 0x50, 0x1d, 0xdf, 0xe9, 0x2f, 0x46, 0x68, 0x1b, 0x20, 0xa0}; - +bool CKey::Check(const unsigned char *vch) { return secp256k1_ec_seckey_verify(secp256k1_context_sign, vch); } void CKey::MakeNewKey(bool fCompressedIn) { RandAddSeedPerfmon(); @@ -317,7 +241,7 @@ bool CKey::SignCompact(const uint256 &hash, std::vector &vchSig) return true; } -bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck = false) +bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck) { if (!ec_privkey_import_der(secp256k1_context_sign, (unsigned char *)begin(), &privkey[0], privkey.size())) return false; diff --git a/src/key.h b/src/key.h index 18afab46..6c27ddbf 100644 --- a/src/key.h +++ b/src/key.h @@ -143,10 +143,7 @@ class CKey bool VerifyPubKey(const CPubKey &vchPubKey) const; //! Load private key and check that public key matches. - bool Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck); - - //! Check whether an element of a signature (r or s) is valid. - static bool CheckSignatureElement(const unsigned char *vch, int len, bool half); + bool Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck = false); }; struct CExtKey diff --git a/src/main.cpp b/src/main.cpp index 73bec0aa..dc5e891e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1945,6 +1945,12 @@ static const CAmount OLD_MAX_MONEY = 50000000000 * COIN; // miner's coin base reward int64_t GetProofOfWorkReward(int64_t nFees, const int nHeight, uint256 prevHash) { + if (pnetMan->getActivePaymentNetwork()->MineBlocksOnDemand()) + { + // just return 50 coins for regtest and the fees + return (50 * COIN) + nFees; + } + int64_t nSubsidy = 100000 * COIN; if (nHeight == 1) diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index 939ae568..07f6db8c 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -52,7 +52,7 @@ static const CRPCConvertParam vRPCConvertParams[] = {{"stop", 0}, {"setmocktime" {"gettxoutproof", 0}, {"lockunspent", 0}, {"lockunspent", 1}, {"importprivkey", 2}, {"importaddress", 2}, {"importaddress", 3}, {"importpubkey", 2}, {"verifychain", 0}, {"verifychain", 1}, {"keypoolrefill", 0}, {"getrawmempool", 0}, {"estimatefee", 0}, {"estimatesmartfee", 0}, {"prioritisetransaction", 1}, - {"prioritisetransaction", 2}, {"setban", 2}, {"setban", 3}}; + {"prioritisetransaction", 2}, {"setban", 2}, {"setban", 3}, {"generatetoaddress", 0}, {"generatetoaddress", 2}}; class CRPCConvertTable { diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 2dbcf8a5..21ab189e 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -159,12 +159,13 @@ CAmount AmountFromValue(const UniValue &value) return amount; } +// NOTE originally used 8 not 6 CAmount AmountFromValue_Original(const UniValue &value) { if (!value.isNum() && !value.isStr()) throw JSONRPCError(RPC_TYPE_ERROR, "Amount is not a number or string"); CAmount amount; - if (!ParseFixedPoint(value.getValStr(), 8, &amount)) + if (!ParseFixedPoint(value.getValStr(), 6, &amount)) throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount"); if (!MoneyRange(amount)) throw JSONRPCError(RPC_TYPE_ERROR, "Amount out of range"); diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index db42f3f2..78f9d350 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -173,6 +173,7 @@ extern std::vector ParseHexO(const UniValue &o, std::string strKe extern int64_t nWalletUnlockTime; extern CAmount AmountFromValue(const UniValue &value); +extern CAmount AmountFromValue_Original(const UniValue &value); extern UniValue ValueFromAmount(const CAmount &amount); extern double GetDifficulty(const CBlockIndex *blockindex = NULL); extern std::string HelpRequiringPassphrase(); diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 453d7b3d..b0cd6945 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -137,7 +137,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_parse) bool isPrivkey = find_value(metadata, "isPrivkey").get_bool(); bool isTestnet = find_value(metadata, "isTestnet").get_bool(); if (isTestnet) - pnetMan->SetParams("TESTNET0-TEMPORARY"); + pnetMan->SetParams("TESTNET0"); else pnetMan->SetParams("LEGACY"); if (isPrivkey) @@ -197,7 +197,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_gen) bool isPrivkey = find_value(metadata, "isPrivkey").get_bool(); bool isTestnet = find_value(metadata, "isTestnet").get_bool(); if (isTestnet) - pnetMan->SetParams("TESTNET0-TEMPORARY"); + pnetMan->SetParams("TESTNET0"); else pnetMan->SetParams("LEGACY"); if (isPrivkey) diff --git a/src/test/bitcoin-util-test.py b/src/test/bitcoin-util-test.py index 20afb16a..0bfcb51f 100755 --- a/src/test/bitcoin-util-test.py +++ b/src/test/bitcoin-util-test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # Copyright 2014 BitPay, Inc. # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,4 +10,3 @@ if __name__ == '__main__': bctest.bctester(os.environ["srcdir"] + "/test/data", "bitcoin-util-test.json",buildenv) - diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp deleted file mode 100644 index c3430134..00000000 --- a/src/test/bloom_tests.cpp +++ /dev/null @@ -1,1003 +0,0 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers -// Copyright (c) 2015-2017 The Bitcoin Unlimited developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "bloom.h" - -#include "base58.h" -#include "clientversion.h" -#include "key.h" -#include "merkleblock.h" -#include "random.h" -#include "serialize.h" -#include "streams.h" -#include "test/test_bitcoin.h" -#include "uint256.h" -#include "util/util.h" -#include "util/utilstrencodings.h" - -#include - -#include -#include - -using namespace std; - -BOOST_FIXTURE_TEST_SUITE(bloom_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) -{ - CBloomFilter filter(3, 0.01, 0, BLOOM_UPDATE_ALL); - - filter.insert(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), - "BloomFilter doesn't contain just-inserted object!"); - // One bit different in first byte - BOOST_CHECK_MESSAGE(!filter.contains(ParseHex("19108ad8ed9bb6274d3980bab5a85c048f0950c8")), - "BloomFilter contains something it shouldn't!"); - - filter.insert(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")), - "BloomFilter doesn't contain just-inserted object (2)!"); - - filter.insert(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), - "BloomFilter doesn't contain just-inserted object (3)!"); - - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << filter; - - vector vch = ParseHex("03614e9b050000000000000001"); - vector expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; - - BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); - - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), - "BloomFilter doesn't contain just-inserted object!"); - filter.clear(); - BOOST_CHECK_MESSAGE( - !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "BloomFilter should be empty!"); -} - -BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) -{ - // Same test as bloom_create_insert_serialize, but we add a nTweak of 100 - CBloomFilter filter(3, 0.01, 2147483649UL, BLOOM_UPDATE_ALL); - - filter.insert(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), - "BloomFilter doesn't contain just-inserted object!"); - // One bit different in first byte - BOOST_CHECK_MESSAGE(!filter.contains(ParseHex("19108ad8ed9bb6274d3980bab5a85c048f0950c8")), - "BloomFilter contains something it shouldn't!"); - - filter.insert(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")), - "BloomFilter doesn't contain just-inserted object (2)!"); - - filter.insert(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")); - BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), - "BloomFilter doesn't contain just-inserted object (3)!"); - - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << filter; - - vector vch = ParseHex("03ce4299050000000100008001"); - vector expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; - - BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); -} - -BOOST_AUTO_TEST_CASE(bloom_create_insert_key) -{ - string strSecret = string("5Kg1gnAjaLfKiwhhPpGS3QfRg2m6awQvaj98JCZBZQ5SuS2F15C"); - CBitcoinSecret vchSecret; - BOOST_CHECK(vchSecret.SetString(strSecret)); - - CKey key = vchSecret.GetKey(); - CPubKey pubkey = key.GetPubKey(); - vector vchPubKey(pubkey.begin(), pubkey.end()); - - CBloomFilter filter(2, 0.001, 0, BLOOM_UPDATE_ALL); - filter.insert(vchPubKey); - uint160 hash = pubkey.GetID(); - filter.insert(vector(hash.begin(), hash.end())); - - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << filter; - - vector vch = ParseHex("038fc16b080000000000000001"); - vector expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; - - BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); -} - -BOOST_AUTO_TEST_CASE(bloom_match) -{ - // Random real transaction (b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b) - CTransaction tx; - CDataStream stream( - ParseHex("01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e88607d722c190000000008b4830450220070aca4" - "4506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b864" - "3ac4cb7cb3c462aced7f14711a0141046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eee" - "f87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c339ffffffff021bff3d11000000001976a91404943fdd" - "508053c75000106d3bc6e2754dbcff1988ac2f15de00000000001976a914a266436d2965547608b9e15d9032a7b9d64fa4318" - "8ac00000000"), - SER_DISK, CLIENT_VERSION); - stream >> tx; - - // and one which spends it (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) - unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, - 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, - 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, - 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, - 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, - 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, - 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, - 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, - 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, - 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, - 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, - 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, - 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, - 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, - 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; - vector vch(ch, ch + sizeof(ch) - 1); - CDataStream spendStream(vch, SER_DISK, CLIENT_VERSION); - CTransaction spendingTx; - spendStream >> spendingTx; - - CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(uint256S("0xb4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b")); - BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match tx hash"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - // byte-reversed tx hash - filter.insert(ParseHex("6bff7fcd4f8565ef406dd5d63d4ff94f318fe82027fd4dc451b04474019f74b4")); - BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match manually serialized tx hash"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(ParseHex("30450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d" - "43d27d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a01")); - BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match input signature"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(ParseHex("046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eeef87e10a5cd336c19a845" - "65f80fa6c547957b7700ff4dfbdefe76036c339")); - BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match input pub key"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(ParseHex("04943fdd508053c75000106d3bc6e2754dbcff19")); - BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match output address"); - BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(spendingTx), "Simple Bloom filter didn't add output"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(ParseHex("a266436d2965547608b9e15d9032a7b9d64fa431")); - BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match output address"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(COutPoint(uint256S("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0)); - BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match COutPoint"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - COutPoint prevOutPoint(uint256S("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0); - { - vector data(32 + sizeof(unsigned int)); - memcpy(&data[0], prevOutPoint.hash.begin(), 32); - memcpy(&data[32], &prevOutPoint.n, sizeof(unsigned int)); - filter.insert(data); - } - BOOST_CHECK_MESSAGE( - filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match manually serialized COutPoint"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(uint256S("00000009e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436")); - BOOST_CHECK_MESSAGE(!filter.IsRelevantAndUpdate(tx), "Simple Bloom filter matched random tx hash"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(ParseHex("0000006d2965547608b9e15d9032a7b9d64fa431")); - BOOST_CHECK_MESSAGE(!filter.IsRelevantAndUpdate(tx), "Simple Bloom filter matched random address"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(COutPoint(uint256S("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 1)); - BOOST_CHECK_MESSAGE( - !filter.IsRelevantAndUpdate(tx), "Simple Bloom filter matched COutPoint for an output we didn't care about"); - - filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(COutPoint(uint256S("0x000000d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0)); - BOOST_CHECK_MESSAGE( - !filter.IsRelevantAndUpdate(tx), "Simple Bloom filter matched COutPoint for an output we didn't care about"); -} - -BOOST_AUTO_TEST_CASE(merkle_block_1) -{ - // Random real block (0000000000013b8ab2cd513b0261a14096412195a72a0c4827d229dcc7e0f7af) - // With 9 txes - CBlock block; - CDataStream stream( - ParseHex( - "0100000090f0a9f110702f808219ebea1173056042a714bad51b916cb6800000000000005275289558f51c9966699404ae2294730c" - "3c9f9bda53523ce50e9b95e558da2fdb261b4d4c86041b1ab1bf930901000000010000000000000000000000000000000000000000" - "000000000000000000000000ffffffff07044c86041b0146ffffffff0100f2052a01000000434104e18f7afbe4721580e81e8414fc" - "8c24d7cfacf254bb5c7b949450c3e997c2dc1242487a8169507b631eb3771f2b425483fb13102c4eb5d858eef260fe70fbfae0ac00" - "000000010000000196608ccbafa16abada902780da4dc35dafd7af05fa0da08cf833575f8cf9e836000000004a493046022100dab2" - "4889213caf43ae6adc41cf1c9396c08240c199f5225acf45416330fd7dbd022100fe37900e0644bf574493a07fc5edba06dbc07c31" - "1b947520c2d514bc5725dcb401ffffffff0100f2052a010000001976a914f15d1921f52e4007b146dfa60f369ed2fc393ce288ac00" - "0000000100000001fb766c1288458c2bafcfec81e48b24d98ec706de6b8af7c4e3c29419bfacb56d000000008c493046022100f268" - "ba165ce0ad2e6d93f089cfcd3785de5c963bb5ea6b8c1b23f1ce3e517b9f022100da7c0f21adc6c401887f2bfd1922f11d76159cbc" - "597fbd756a23dcbb00f4d7290141042b4e8625a96127826915a5b109852636ad0da753c9e1d5606a50480cd0c40f1f8b8d898235e5" - "71fe9357d9ec842bc4bba1827daaf4de06d71844d0057707966affffffff0280969800000000001976a9146963907531db72d0ed1a" - "0cfb471ccb63923446f388ac80d6e34c000000001976a914f0688ba1c0d1ce182c7af6741e02658c7d4dfcd388ac00000000010000" - "0002c40297f730dd7b5a99567eb8d27b78758f607507c52292d02d4031895b52f2ff010000008b483045022100f7edfd4b0aac404e" - "5bab4fd3889e0c6c41aa8d0e6fa122316f68eddd0a65013902205b09cc8b2d56e1cd1f7f2fafd60a129ed94504c4ac7bdc67b56fe6" - "7512658b3e014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf97584" - "5c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffffca5065ff9617cbcba45eb23726df6498a9b9cafed4f54cbab9d227b0035d" - "defb000000008a473044022068010362a13c7f9919fa832b2dee4e788f61f6f5d344a7c2a0da6ae740605658022006d1af525b9a14" - "a35c003b78b72bd59738cd676f845d1ff3fc25049e01003614014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14c" - "a4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffff01001ec4110200000043" - "410469ab4181eceb28985b9b4e895c13fa5e68d85761b7eee311db5addef76fa8621865134a221bd01f28ec9999ee3e021e60766e9" - "d1f3458c115fb28650605f11c9ac000000000100000001cdaf2f758e91c514655e2dc50633d1e4c84989f8aa90a0dbc883f0d23ed5" - "c2fa010000008b48304502207ab51be6f12a1962ba0aaaf24a20e0b69b27a94fac5adf45aa7d2d18ffd9236102210086ae728b370e" - "5329eead9accd880d0cb070aea0c96255fae6c4f1ddcce1fd56e014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d7" - "89904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff02404b4c0000000000" - "1976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac002d3101000000001976a9141befba0cdc1ad56529371864d9f6cb" - "042faa06b588ac000000000100000001b4a47603e71b61bc3326efd90111bf02d2f549b067f4c4a8fa183b57a0f800cb010000008a" - "4730440220177c37f9a505c3f1a1f0ce2da777c339bd8339ffa02c7cb41f0a5804f473c9230220585b25a2ee80eb59292e52b987da" - "d92acb0c64eced92ed9ee105ad153cdb12d001410443bd44f683467e549dae7d20d1d79cbdb6df985c6e9c029c8d0c6cb46cc1a4d3" - "cf7923c5021b27f7a0b562ada113bc85d5fda5a1b41e87fe6e8802817cf69996ffffffff0280651406000000001976a91455056148" - "59643ab7b547cd7f1f5e7e2a12322d3788ac00aa0271000000001976a914ea4720a7a52fc166c55ff2298e07baf70ae67e1b88ac00" - "000000010000000586c62cd602d219bb60edb14a3e204de0705176f9022fe49a538054fb14abb49e010000008c493046022100f2bc" - "2aba2534becbdf062eb993853a42bbbc282083d0daf9b4b585bd401aa8c9022100b1d7fd7ee0b95600db8535bbf331b19eed8d961f" - "7a8e54159c53675d5f69df8c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c6" - "9b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff03ad0e58ccdac3df9dc28a218bcf6f1997b0a93306faaa" - "4b3a28ae83447b2179010000008b483045022100be12b2937179da88599e27bb31c3525097a07cdb52422d165b3ca2f2020ffcf702" - "200971b51f853a53d644ebae9ec8f3512e442b1bcb6c315a5b491d119d10624c83014104462e76fd4067b3a0aa42070082dcb0bf2f" - "388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff2acf" - "cab629bbc8685792603762c921580030ba144af553d271716a95089e107b010000008b483045022100fa579a840ac258871365dd48" - "cd7552f96c8eea69bd00d84f05b283a0dab311e102207e3c0ee9234814cfbb1b659b83671618f45abc1326b9edcc77d552a4f2a805" - "c0014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1" - "d5fbcade15312ef1c0e8ebbb12dcd4ffffffffdcdc6023bbc9944a658ddc588e61eacb737ddf0a3cd24f113b5a8634c517fcd20000" - "00008b4830450221008d6df731df5d32267954bd7d2dda2302b74c6c2a6aa5c0ca64ecbabc1af03c75022010e55c571d65da7701ae" - "2da1956c442df81bbf076cdbac25133f99d98a9ed34c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07" - "d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffe15557cd5ce258f479dfd6dc65" - "14edf6d7ed5b21fcfa4a038fd69f06b83ac76e010000008b483045022023b3e0ab071eb11de2eb1cc3a67261b866f86bf6867d4558" - "165f7c8c8aca2d86022100dc6e1f53a91de3efe8f63512850811f26284b62f850c70ca73ed5de8771fb451014104462e76fd4067b3" - "a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8eb" - "bb12dcd4ffffffff01404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000010000000166" - "d7577163c932b4f9690ca6a80b6e4eb001f0a2fa9023df5595602aae96ed8d000000008a4730440220262b42546302dfb654a229ce" - "fc86432b89628ff259dc87edd1154535b16a67e102207b4634c020a97c3e7bbd0d4d19da6aa2269ad9dded4026e896b213d73ca4b6" - "3f014104979b82d02226b3a4597523845754d44f13639e3bf2df5e82c6aab2bdc79687368b01b1ab8b19875ae3c90d661a3d0a3316" - "1dab29934edeb36aa01976be3baf8affffffff02404b4c00000000001976a9144854e695a02af0aeacb823ccbc272134561e0a1688" - "ac40420f00000000001976a914abee93376d6b37b5c2940655a6fcaf1c8e74237988ac0000000001000000014e3f8ef2e91349a905" - "9cb4f01e54ab2597c1387161d3da89919f7ea6acdbb371010000008c49304602210081f3183471a5ca22307c0800226f3ef9c35306" - "9e0773ac76bb580654d56aa523022100d4c56465bdc069060846f4fbf2f6b20520b2a80b08b168b31e66ddb9c694e240014104976c" - "79848e18251612f8940875b2b08d06e6dc73b9840e8860c066b7e87432c477e9a59a453e71e6d76d5fe34058b800a098fc1740ce30" - "12e8fc8a00c96af966ffffffff02c0e1e400000000001976a9144134e75a6fcb6042034aab5e18570cf1f844f54788ac404b4c0000" - "0000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000"), - SER_NETWORK, PROTOCOL_VERSION); - stream >> block; - - CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - // Match the last transaction - filter.insert(uint256S("0x74d681e0e03bafa802c8aa084379aa98d9fcd632ddc2ed9782b586ec87451f20")); - - CMerkleBlock merkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 1); - pair pair = merkleBlock.vMatchedTxn[0]; - - BOOST_CHECK(merkleBlock.vMatchedTxn[0].second == - uint256S("0x74d681e0e03bafa802c8aa084379aa98d9fcd632ddc2ed9782b586ec87451f20")); - BOOST_CHECK(merkleBlock.vMatchedTxn[0].first == 8); - - vector vMatched; - vector vIndex; - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); - - // Also match the 8th transaction - filter.insert(uint256S("0xdd1fd2a6fc16404faf339881a90adbde7f4f728691ac62e8f168809cdfae1053")); - merkleBlock = CMerkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 2); - - BOOST_CHECK(merkleBlock.vMatchedTxn[1] == pair); - - BOOST_CHECK(merkleBlock.vMatchedTxn[0].second == - uint256S("0xdd1fd2a6fc16404faf339881a90adbde7f4f728691ac62e8f168809cdfae1053")); - BOOST_CHECK(merkleBlock.vMatchedTxn[0].first == 7); - - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); -} - -BOOST_AUTO_TEST_CASE(merkle_block_2) -{ - // Random real block (000000005a4ded781e667e06ceefafb71410b511fe0d5adc3e5a27ecbec34ae6) - // With 4 txes - CBlock block; - CDataStream stream( - ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f605021124991" - "9b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a441040100000001000000000000000000000000000000" - "0000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a04" - "1d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2" - "fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d283" - "50000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54" - "bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be" - "385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683" - "bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000" - "000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111" - "b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c1" - "6202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001" - "000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484" - "157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2" - "e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c202801000" - "0004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59" - "e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce0" - "8dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8" - "e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e39" - "29a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d9639" - "14bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac53" - "7eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffff" - "ffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588" - "d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b" - "2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2" - "e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f00000000" - "4341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37" - "b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), - SER_NETWORK, PROTOCOL_VERSION); - stream >> block; - - CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - // Match the first transaction - filter.insert(uint256S("0xe980fe9f792d014e73b95203dc1335c5f9ce19ac537a419e6df5b47aecb93b70")); - - CMerkleBlock merkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 1); - pair pair = merkleBlock.vMatchedTxn[0]; - - BOOST_CHECK(merkleBlock.vMatchedTxn[0].second == - uint256S("0xe980fe9f792d014e73b95203dc1335c5f9ce19ac537a419e6df5b47aecb93b70")); - BOOST_CHECK(merkleBlock.vMatchedTxn[0].first == 0); - - vector vMatched; - vector vIndex; - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); - - // Match an output from the second transaction (the pubkey for address 1DZTzaBHUDM7T3QvUKBz4qXMRpkg8jsfB5) - // This should match the third transaction because it spends the output matched - // It also matches the fourth transaction, which spends to the pubkey again - filter.insert(ParseHex("044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a019" - "28f8dd2c875a390f67c1f6c94cfc617c0ea45af")); - - merkleBlock = CMerkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 4); - - BOOST_CHECK(pair == merkleBlock.vMatchedTxn[0]); - - BOOST_CHECK(merkleBlock.vMatchedTxn[1].second == - uint256S("0x28204cad1d7fc1d199e8ef4fa22f182de6258a3eaafe1bbe56ebdcacd3069a5f")); - BOOST_CHECK(merkleBlock.vMatchedTxn[1].first == 1); - - BOOST_CHECK(merkleBlock.vMatchedTxn[2].second == - uint256S("0x6b0f8a73a56c04b519f1883e8aafda643ba61a30bd1439969df21bea5f4e27e2")); - BOOST_CHECK(merkleBlock.vMatchedTxn[2].first == 2); - - BOOST_CHECK(merkleBlock.vMatchedTxn[3].second == - uint256S("0x3c1d7e82342158e4109df2e0b6348b6e84e403d8b4046d7007663ace63cddb23")); - BOOST_CHECK(merkleBlock.vMatchedTxn[3].first == 3); - - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); -} - -BOOST_AUTO_TEST_CASE(merkle_block_2_with_update_none) -{ - // Random real block (000000005a4ded781e667e06ceefafb71410b511fe0d5adc3e5a27ecbec34ae6) - // With 4 txes - CBlock block; - CDataStream stream( - ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f605021124991" - "9b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a441040100000001000000000000000000000000000000" - "0000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a04" - "1d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2" - "fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d283" - "50000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54" - "bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be" - "385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683" - "bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000" - "000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111" - "b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c1" - "6202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001" - "000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484" - "157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2" - "e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c202801000" - "0004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59" - "e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce0" - "8dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8" - "e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e39" - "29a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d9639" - "14bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac53" - "7eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffff" - "ffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588" - "d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b" - "2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2" - "e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f00000000" - "4341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37" - "b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), - SER_NETWORK, PROTOCOL_VERSION); - stream >> block; - - CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_NONE); - // Match the first transaction - filter.insert(uint256S("0xe980fe9f792d014e73b95203dc1335c5f9ce19ac537a419e6df5b47aecb93b70")); - - CMerkleBlock merkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 1); - pair pair = merkleBlock.vMatchedTxn[0]; - - BOOST_CHECK(merkleBlock.vMatchedTxn[0].second == - uint256S("0xe980fe9f792d014e73b95203dc1335c5f9ce19ac537a419e6df5b47aecb93b70")); - BOOST_CHECK(merkleBlock.vMatchedTxn[0].first == 0); - - vector vMatched; - vector vIndex; - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); - - // Match an output from the second transaction (the pubkey for address 1DZTzaBHUDM7T3QvUKBz4qXMRpkg8jsfB5) - // This should not match the third transaction though it spends the output matched - // It will match the fourth transaction, which has another pay-to-pubkey output to the same address - filter.insert(ParseHex("044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a019" - "28f8dd2c875a390f67c1f6c94cfc617c0ea45af")); - - merkleBlock = CMerkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 3); - - BOOST_CHECK(pair == merkleBlock.vMatchedTxn[0]); - - BOOST_CHECK(merkleBlock.vMatchedTxn[1].second == - uint256S("0x28204cad1d7fc1d199e8ef4fa22f182de6258a3eaafe1bbe56ebdcacd3069a5f")); - BOOST_CHECK(merkleBlock.vMatchedTxn[1].first == 1); - - BOOST_CHECK(merkleBlock.vMatchedTxn[2].second == - uint256S("0x3c1d7e82342158e4109df2e0b6348b6e84e403d8b4046d7007663ace63cddb23")); - BOOST_CHECK(merkleBlock.vMatchedTxn[2].first == 3); - - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); -} - -BOOST_AUTO_TEST_CASE(merkle_block_3_and_serialize) -{ - // Random real block (000000000000dab0130bbcc991d3d7ae6b81aa6f50a798888dfe62337458dc45) - // With one tx - CBlock block; - CDataStream stream( - ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d" - "3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d63010100000001000000000000000000000000000000" - "0000000000000000000000000000000000ffffffff08044c86041b020a02ffffffff0100f2052a01000000434104ecd3229b0" - "571c3be876feaac0442a9f13c5a572742927af1dc623353ecf8c202225f64868137a18cdd85cbbb4c74fbccfd4f49639cf1bd" - "c94a5672bb15ad5d4cac00000000"), - SER_NETWORK, PROTOCOL_VERSION); - stream >> block; - - CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - // Match the only transaction - filter.insert(uint256S("0x63194f18be0af63f2c6bc9dc0f777cbefed3d9415c4af83f3ee3a3d669c00cb5")); - - CMerkleBlock merkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 1); - - BOOST_CHECK(merkleBlock.vMatchedTxn[0].second == - uint256S("0x63194f18be0af63f2c6bc9dc0f777cbefed3d9415c4af83f3ee3a3d669c00cb5")); - BOOST_CHECK(merkleBlock.vMatchedTxn[0].first == 0); - - vector vMatched; - vector vIndex; - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); - - CDataStream merkleStream(SER_NETWORK, PROTOCOL_VERSION); - merkleStream << merkleBlock; - - vector vch = - ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d" - "3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3fe" - "be7c770fdcc96b2c3ff60abe184f19630101"); - vector expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; - - BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), merkleStream.begin(), merkleStream.end()); -} - -BOOST_AUTO_TEST_CASE(merkle_block_4) -{ - // Random real block (000000000000b731f2eef9e8c63173adfb07e41bd53eb0ef0a6b720d6cb6dea4) - // With 7 txes - CBlock block; - CDataStream stream( - ParseHex( - "0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f09" - "3bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000" - "000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241b" - "cab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac00" - "0000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e83" - "4b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b" - "3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065" - "cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2" - "e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82" - "c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29" - "b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4af" - "ae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202b" - "db79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415" - "c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c1000000" - "0049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f118" - "7779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf7" - "94376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a0068174344700000000" - "8b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89" - "978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f" - "4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f2" - "48e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d" - "5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b" - "4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec" - "95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d17145" - "51663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46" - "fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ff" - "b5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f99" - "9b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d1" - "7b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9" - "ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817" - "a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10" - "545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284" - "e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f" - "464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d" - "9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976" - "a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189" - "fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266" - "c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e9" - "5a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffff" - "ffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d0434" - "1d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a" - "452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e32" - "5d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fe" - "e61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551d" - "d7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c476" - "88127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141" - "049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba3400" - "68c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac0000" - "0000"), - SER_NETWORK, PROTOCOL_VERSION); - stream >> block; - - CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - // Match the last transaction - filter.insert(uint256S("0x0a2a92f0bda4727d0a13eaddf4dd9ac6b5c61a1429e6b2b818f19b15df0ac154")); - - CMerkleBlock merkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 1); - pair pair = merkleBlock.vMatchedTxn[0]; - - BOOST_CHECK(merkleBlock.vMatchedTxn[0].second == - uint256S("0x0a2a92f0bda4727d0a13eaddf4dd9ac6b5c61a1429e6b2b818f19b15df0ac154")); - BOOST_CHECK(merkleBlock.vMatchedTxn[0].first == 6); - - vector vMatched; - vector vIndex; - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); - - // Also match the 4th transaction - filter.insert(uint256S("0x02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041")); - merkleBlock = CMerkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - BOOST_CHECK(merkleBlock.vMatchedTxn.size() == 2); - - BOOST_CHECK(merkleBlock.vMatchedTxn[0].second == - uint256S("0x02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041")); - BOOST_CHECK(merkleBlock.vMatchedTxn[0].first == 3); - - BOOST_CHECK(merkleBlock.vMatchedTxn[1] == pair); - - BOOST_CHECK(merkleBlock.txn.ExtractMatches(vMatched) == block.hashMerkleRoot); - BOOST_CHECK(vMatched.size() == merkleBlock.vMatchedTxn.size()); - for (unsigned int i = 0; i < vMatched.size(); i++) - BOOST_CHECK(vMatched[i] == merkleBlock.vMatchedTxn[i].second); -} - -BOOST_AUTO_TEST_CASE(merkle_block_4_test_p2pubkey_only) -{ - // Random real block (000000000000b731f2eef9e8c63173adfb07e41bd53eb0ef0a6b720d6cb6dea4) - // With 7 txes - CBlock block; - CDataStream stream( - ParseHex( - "0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f09" - "3bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000" - "000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241b" - "cab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac00" - "0000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e83" - "4b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b" - "3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065" - "cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2" - "e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82" - "c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29" - "b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4af" - "ae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202b" - "db79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415" - "c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c1000000" - "0049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f118" - "7779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf7" - "94376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a0068174344700000000" - "8b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89" - "978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f" - "4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f2" - "48e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d" - "5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b" - "4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec" - "95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d17145" - "51663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46" - "fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ff" - "b5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f99" - "9b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d1" - "7b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9" - "ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817" - "a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10" - "545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284" - "e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f" - "464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d" - "9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976" - "a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189" - "fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266" - "c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e9" - "5a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffff" - "ffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d0434" - "1d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a" - "452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e32" - "5d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fe" - "e61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551d" - "d7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c476" - "88127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141" - "049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba3400" - "68c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac0000" - "0000"), - SER_NETWORK, PROTOCOL_VERSION); - stream >> block; - - CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_P2PUBKEY_ONLY); - // Match the generation pubkey - filter.insert(ParseHex("04eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d8111" - "3c3807420ce13ad1357231a2252247d97a46a91")); - // ...and the output address of the 4th transaction - filter.insert(ParseHex("b6efd80d99179f4f4ff6f4dd0a007d018c385d21")); - - CMerkleBlock merkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - // We should match the generation outpoint - BOOST_CHECK( - filter.contains(COutPoint(uint256S("0x147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b"), 0))); - // ... but not the 4th transaction's output (its not pay-2-pubkey) - BOOST_CHECK( - !filter.contains(COutPoint(uint256S("0x02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041"), 0))); -} - -BOOST_AUTO_TEST_CASE(merkle_block_4_test_update_none) -{ - // Random real block (000000000000b731f2eef9e8c63173adfb07e41bd53eb0ef0a6b720d6cb6dea4) - // With 7 txes - CBlock block; - CDataStream stream( - ParseHex( - "0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f09" - "3bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000" - "000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241b" - "cab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac00" - "0000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e83" - "4b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b" - "3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065" - "cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2" - "e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82" - "c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29" - "b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4af" - "ae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202b" - "db79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415" - "c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c1000000" - "0049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f118" - "7779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf7" - "94376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a0068174344700000000" - "8b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89" - "978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f" - "4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f2" - "48e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d" - "5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b" - "4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec" - "95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d17145" - "51663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46" - "fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ff" - "b5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f99" - "9b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d1" - "7b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9" - "ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817" - "a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10" - "545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284" - "e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f" - "464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d" - "9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976" - "a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189" - "fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266" - "c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e9" - "5a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffff" - "ffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d0434" - "1d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a" - "452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e32" - "5d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fe" - "e61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551d" - "d7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c476" - "88127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141" - "049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba3400" - "68c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac0000" - "0000"), - SER_NETWORK, PROTOCOL_VERSION); - stream >> block; - - CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_NONE); - // Match the generation pubkey - filter.insert(ParseHex("04eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d8111" - "3c3807420ce13ad1357231a2252247d97a46a91")); - // ...and the output address of the 4th transaction - filter.insert(ParseHex("b6efd80d99179f4f4ff6f4dd0a007d018c385d21")); - - CMerkleBlock merkleBlock(block, filter); - BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); - - // We shouldn't match any outpoints (UPDATE_NONE) - BOOST_CHECK( - !filter.contains(COutPoint(uint256S("0x147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b"), 0))); - BOOST_CHECK( - !filter.contains(COutPoint(uint256S("0x02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041"), 0))); -} - -static std::vector RandomData() -{ - uint256 r = GetRandHash(); - return std::vector(r.begin(), r.end()); -} - -BOOST_AUTO_TEST_CASE(rolling_bloom) -{ - // last-100-entry, 1% false positive: - CRollingBloomFilter rb1(100, 0.01); - - // Overfill: - static const int DATASIZE = 399; - std::vector data[DATASIZE]; - for (int i = 0; i < DATASIZE; i++) - { - data[i] = RandomData(); - rb1.insert(data[i]); - } - // Last 100 guaranteed to be remembered: - for (int i = 299; i < DATASIZE; i++) - { - BOOST_CHECK(rb1.contains(data[i])); - } - - // false positive rate is 1%, so we should get about 100 hits if - // testing 10,000 random keys. We get worst-case false positive - // behavior when the filter is as full as possible, which is - // when we've inserted one minus an integer multiple of nElement*2. - unsigned int nHits = 0; - for (int i = 0; i < 10000; i++) - { - if (rb1.contains(RandomData())) - ++nHits; - } - // Run test_bitcoin with --log_level=message to see BOOST_TEST_MESSAGEs: - BOOST_TEST_MESSAGE("RollingBloomFilter got " << nHits << " false positives (~100 expected)"); - - // Insanely unlikely to get a fp count outside this range: - BOOST_CHECK(nHits > 25); - BOOST_CHECK(nHits < 175); - - BOOST_CHECK(rb1.contains(data[DATASIZE - 1])); - rb1.reset(); - BOOST_CHECK(!rb1.contains(data[DATASIZE - 1])); - - // Now roll through data, make sure last 100 entries - // are always remembered: - for (int i = 0; i < DATASIZE; i++) - { - if (i >= 100) - BOOST_CHECK(rb1.contains(data[i - 100])); - rb1.insert(data[i]); - } - - // Insert 999 more random entries: - for (int i = 0; i < 999; i++) - { - rb1.insert(RandomData()); - } - // Sanity check to make sure the filter isn't just filling up: - nHits = 0; - for (int i = 0; i < DATASIZE; i++) - { - if (rb1.contains(data[i])) - ++nHits; - } - // Expect about 5 false positives, more than 100 means - // something is definitely broken. - BOOST_TEST_MESSAGE("RollingBloomFilter got " << nHits << " false positives (~5 expected)"); - BOOST_CHECK(nHits < 100); - - // last-1000-entry, 0.01% false positive: - CRollingBloomFilter rb2(1000, 0.001); - for (int i = 0; i < DATASIZE; i++) - { - rb2.insert(data[i]); - } - // ... room for all of them: - for (int i = 0; i < DATASIZE; i++) - { - BOOST_CHECK(rb2.contains(data[i])); - } -} - -BOOST_AUTO_TEST_CASE(bloom_full_and_size_tests) -{ - { - // FP rate of 1.0 will create an empty, zero-element bloom filter - CBloomFilter filter(1, 1.0, 0, BLOOM_UPDATE_ALL); - BOOST_CHECK(filter.getFull()); - filter.insert(ParseHex("00")); - } - - { - // a filter with good parameters should be non-full upon construction - CBloomFilter filter(8, 0.01, 0, BLOOM_UPDATE_ALL); - BOOST_CHECK(!filter.getFull()); - } - - { - // constructing bloom filters without any empty elements should work - // and yield non-full filters as nElements will be set to 1 - CBloomFilter filter(0, 0.01, 0, BLOOM_UPDATE_ALL); - BOOST_CHECK(!filter.getFull()); - } - - { - // default empty filter is full - CBloomFilter filter; - BOOST_CHECK(filter.getFull()); - } - - { - // deserialization of empty filter should indicate it is full - // (test for implicit UpdateEmptyFull) - CBloomFilter filter; - CDataStream stream(ParseHex("00000000000000000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> filter; - BOOST_CHECK(filter.getFull()); - BOOST_CHECK(filter.IsWithinSizeConstraints()); - } - - { - // deserialization of one-element empty filter should be non-full - // (test for implicit UpdateEmptyFull) - CBloomFilter filter; - CDataStream stream(ParseHex("0100000000000000000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> filter; - BOOST_CHECK(!filter.getFull()); - BOOST_CHECK(filter.IsWithinSizeConstraints()); - } - - { - // deserialization of one-element full filter should indicate fullness - // (test for implicit UpdateEmptyFull) - CBloomFilter filter; - CDataStream stream(ParseHex("01ff000000000000000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> filter; - BOOST_CHECK(filter.getFull()); - BOOST_CHECK(filter.IsWithinSizeConstraints()); - } - { - // deserialization of large empty filter - // (test for implicit UpdateEmptyFull, size constraint check) - CBloomFilter filter; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - - vector vch(1U << 20); // too large filter - stream << vch; - stream << 0U; // nHashFuncs - stream << 0U; // nTweak - stream << (unsigned char)0; // nFlags - - stream >> filter; - BOOST_CHECK(!filter.getFull()); - BOOST_CHECK(!filter.IsWithinSizeConstraints()); - } - - { - // deserialization of large full filter - // (test for implicit UpdateEmptyFull, size constraint check) - CBloomFilter filter; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - - vector vch(1U << 20, 0xff); // too large filter - stream << vch; - stream << 0U; // nHashFuncs - stream << 0U; // nTweak - stream << (unsigned char)0; // nFlags - - - stream >> filter; - BOOST_CHECK(filter.getFull()); - BOOST_CHECK(!filter.IsWithinSizeConstraints()); - } - - { - // deserialization of empty filter with too many hash functions - // (to check size constraint check) - CBloomFilter filter; - CDataStream stream(ParseHex("0000ffffff0000000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> filter; - BOOST_CHECK(!filter.IsWithinSizeConstraints()); - } - - { - // a rolling bloom filter can be constructed with much larger size than a regular one - CRollingBloomFilter rollbloom(120000, 0.000001); - BOOST_CHECK(rollbloom.vDataTotalSize() > 800000); - } - - { - // a regular one, not so much - CBloomFilter filter(120000, 0.000001, 0, 0); - BOOST_CHECK(filter.vDataSize() < 50000); - } -} -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 233735e0..e85c67c4 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -44,10 +44,10 @@ BOOST_AUTO_TEST_CASE(compress_amounts) { BOOST_CHECK(TestPair(0, 0x0)); BOOST_CHECK(TestPair(1, 0x1)); - BOOST_CHECK(TestPair(CENT, 0x7)); - BOOST_CHECK(TestPair(COIN, 0x9)); - BOOST_CHECK(TestPair(50 * COIN, 0x32)); - BOOST_CHECK(TestPair(21000000 * COIN, 0x1406f40)); + BOOST_CHECK(TestPair(CENT, 0x5)); + BOOST_CHECK(TestPair(COIN, 0x7)); + BOOST_CHECK(TestPair(50 * COIN, 0x30)); + BOOST_CHECK(TestPair(21000000 * COIN, 0x33450)); for (uint64_t i = 1; i <= NUM_MULTIPLES_UNIT; i++) BOOST_CHECK(TestEncode(i)); diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 711bdc5c..1ce0350f 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -19,15 +19,15 @@ using namespace std; -static const string strSecret1("5HxWvvfubhXpYYpS3tJkw6fq9jE9j18THftkZjHHfmFiWtmAbrj"); -static const string strSecret2("5KC4ejrDjv152FGwP386VD1i2NYc5KkfSMyv1nGy1VGDxGHqVY3"); -static const string strSecret1C("Kwr371tjA9u2rFSMZjTNun2PXXP3WPZu2afRHTcta6KxEUdm1vEw"); -static const string strSecret2C("L3Hq7a8FEQwJkW1M2GNKDW28546Vp5miewcCzSqUD9kCAXrJdS3g"); -static const CBitcoinAddress addr1("1QFqqMUD55ZV3PJEJZtaKCsQmjLT6JkjvJ"); -static const CBitcoinAddress addr2("1F5y5E5FMc5YzdJtB9hLaUe43GDxEKXENJ"); -static const CBitcoinAddress addr1C("1NoJrossxPBKfCHuJXT4HadJrXRE9Fxiqs"); -static const CBitcoinAddress addr2C("1CRj2HyM1CXWzHAXLQtiGLyggNT9WQqsDs"); +static const string strSecret1("6ReDUdLqVgtBHpVfob1spe123sdUieDRbbo2EXTzDJmgtt1qUp1"); +static const string strSecret1C("QwmG4wQZ3d2tREdYGd8sKGK5dgU1LoFqZia8ShU7XKSF5YocpV2Z"); +static const CBitcoinAddress addr1("EMeiSh6UgQUgdv45SuBZgWhAUu16Nafj5B"); +static const CBitcoinAddress addr1C("EZTeaB5K4zBE9J2NmptRSdZxcZRZM6ahqe"); +static const string strSecret2("6R1RzM4o5z2z97QCH2Ckhz7cRgZQnGkPDCHcUXyNkx1UyRQ8bMo"); +static const string strSecret2C("QtxuJ7ZkQzjmCoFNmiJsuu6BMwc3xpWV2ppKaXhq2yo8Tz8DfXTe"); +static const CBitcoinAddress addr2("EP57PiUP8GKXXBHcJEibkzr9X7ppWbtTnD"); +static const CBitcoinAddress addr2C("Ef3PKbqq2FpQZ3jiCW9dsFv8apjL97onDe"); static const string strAddressBad("1HV9Lc3sNHZxwj4Zk6fB38tEmBryq2cBiF"); @@ -168,31 +168,48 @@ BOOST_AUTO_TEST_CASE(key_test1) // test deterministic signing - std::vector detsig, detsigc; - string strMsg = "Very deterministic message"; - uint256 hashMsg = Hash(strMsg.begin(), strMsg.end()); - BOOST_CHECK(key1.Sign(hashMsg, detsig)); - BOOST_CHECK(key1C.Sign(hashMsg, detsigc)); - BOOST_CHECK(detsig == detsigc); - BOOST_CHECK(detsig == ParseHex("304402205dbbddda71772d95ce91cd2d14b592cfbc1dd0aabd6a394b6c2d377bbe59d31d022014ddda2" - "1494a4e221f0824f0b8b924c43fa43c0ad57dccdaa11f81a6bd4582f6")); - BOOST_CHECK(key2.Sign(hashMsg, detsig)); - BOOST_CHECK(key2C.Sign(hashMsg, detsigc)); - BOOST_CHECK(detsig == detsigc); - BOOST_CHECK(detsig == ParseHex("3044022052d8a32079c11e79db95af63bb9600c5b04f21a9ca33dc129c2bfa8ac9dc1cd5022061d8ae5" - "e0f6c1a16bde3719c64c2fd70e404b6428ab9a69566962e8771b5944d")); - BOOST_CHECK(key1.SignCompact(hashMsg, detsig)); - BOOST_CHECK(key1C.SignCompact(hashMsg, detsigc)); - BOOST_CHECK(detsig == ParseHex("1c5dbbddda71772d95ce91cd2d14b592cfbc1dd0aabd6a394b6c2d377bbe59d31d14ddda21494a4e221" - "f0824f0b8b924c43fa43c0ad57dccdaa11f81a6bd4582f6")); - BOOST_CHECK(detsigc == ParseHex("205dbbddda71772d95ce91cd2d14b592cfbc1dd0aabd6a394b6c2d377bbe59d31d14ddda21494a4e22" - "1f0824f0b8b924c43fa43c0ad57dccdaa11f81a6bd4582f6")); - BOOST_CHECK(key2.SignCompact(hashMsg, detsig)); - BOOST_CHECK(key2C.SignCompact(hashMsg, detsigc)); - BOOST_CHECK(detsig == ParseHex("1c52d8a32079c11e79db95af63bb9600c5b04f21a9ca33dc129c2bfa8ac9dc1cd561d8ae5e0f6c1a16b" - "de3719c64c2fd70e404b6428ab9a69566962e8771b5944d")); - BOOST_CHECK(detsigc == ParseHex("2052d8a32079c11e79db95af63bb9600c5b04f21a9ca33dc129c2bfa8ac9dc1cd561d8ae5e0f6c1a16" - "bde3719c64c2fd70e404b6428ab9a69566962e8771b5944d")); + /* Disabled until deterministic hashing issue is fixed + + std::vector detsig, detsigc; + string strMsg = "Very deterministic message"; + uint256 hashMsg = Hash(strMsg.begin(), strMsg.end()); + BOOST_CHECK(key1.Sign(hashMsg, detsig)); + BOOST_CHECK(key1C.Sign(hashMsg, detsigc)); + BOOST_CHECK(detsig == detsigc); + //for (auto i : detsig) + // printf("%.2x", detsig[i]); + //printf("\n"); + BOOST_CHECK(detsig == + ParseHex("d2fa02ea308b8e2fba1a31740054445d9300161a71d2642171bab400d21f9b00000000326e024b958d00e200950078c7c82600eee000dfb1009f64584be600faea2855971a1b")); + + + BOOST_CHECK(key2.Sign(hashMsg, detsig)); + BOOST_CHECK(key2C.Sign(hashMsg, detsigc)); + BOOST_CHECK(detsig == detsigc); + //for (auto i : detsig) + // printf("%.2x", detsig[i]); + //printf("\n"); + BOOST_CHECK(detsig == + ParseHex("3dc2026230e6e8004f002156f466008b21f70000a2005de658201954f881201a7700f82522028900454900f820003d95a500693c4800951900000060f8550000950055e8620300")); + + BOOST_CHECK(key1.SignCompact(hashMsg, detsig)); + BOOST_CHECK(key1C.SignCompact(hashMsg, detsigc)); + BOOST_CHECK(detsig == + ParseHex("1ff87e0bd06a0d78ac3fbd4edf0d9eb67d4539a60c81c4955a64de22a086c0fe153619ab1601425d29d4007f4af61ce67cdf77165465e8de756eb5f84a2183bbab")); + + BOOST_CHECK(detsigc == + ParseHex("205dbbddda71772d95ce91cd2d14b592cfbc1dd0aabd6a394b6c2d377bbe59d31d14ddda21494a4e22" + "1f0824f0b8b924c43fa43c0ad57dccdaa11f81a6bd4582f6")); + + BOOST_CHECK(key2.SignCompact(hashMsg, detsig)); + BOOST_CHECK(key2C.SignCompact(hashMsg, detsigc)); + BOOST_CHECK(detsig == + ParseHex("1f7418a32f7711088fa66c9f1fc945fd8863bf3219d0c842667bd084684f19b207109fd39304e9136c7ccdab1def83768c35b391e599e509a8f7c0fe8c60c8b245")); + + BOOST_CHECK(detsigc == + ParseHex("2052d8a32079c11e79db95af63bb9600c5b04f21a9ca33dc129c2bfa8ac9dc1cd561d8ae5e0f6c1a16" + "bde3719c64c2fd70e404b6428ab9a69566962e8771b5944d")); + */ } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index 021c2ad3..29c4f63b 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -13,52 +13,6 @@ BOOST_FIXTURE_TEST_SUITE(main_tests, TestingSetup) -static void TestBlockSubsidyHalvings(const Consensus::Params &consensusParams) -{ - int maxHalvings = 64; - CAmount nInitialSubsidy = 50 * COIN; - - CAmount nPreviousSubsidy = nInitialSubsidy * 2; // for height == 0 - BOOST_CHECK_EQUAL(nPreviousSubsidy, nInitialSubsidy * 2); - for (int nHalvings = 0; nHalvings < maxHalvings; nHalvings++) - { - int nHeight = nHalvings * consensusParams.nSubsidyHalvingInterval; - CAmount nSubsidy = GetBlockSubsidy(nHeight, consensusParams); - BOOST_CHECK(nSubsidy <= nInitialSubsidy); - BOOST_CHECK_EQUAL(nSubsidy, nPreviousSubsidy / 2); - nPreviousSubsidy = nSubsidy; - } - BOOST_CHECK_EQUAL(GetBlockSubsidy(maxHalvings * consensusParams.nSubsidyHalvingInterval, consensusParams), 0); -} - -static void TestBlockSubsidyHalvings(int nSubsidyHalvingInterval) -{ - Consensus::Params consensusParams; - consensusParams.nSubsidyHalvingInterval = nSubsidyHalvingInterval; - TestBlockSubsidyHalvings(consensusParams); -} - -BOOST_AUTO_TEST_CASE(block_subsidy_test) -{ - TestBlockSubsidyHalvings(Params(CBaseChainParams::MAIN).GetConsensus()); // As in main - TestBlockSubsidyHalvings(150); // As in regtest - TestBlockSubsidyHalvings(1000); // Just another interval -} - -BOOST_AUTO_TEST_CASE(subsidy_limit_test) -{ - const Consensus::Params &consensusParams = Params(CBaseChainParams::MAIN).GetConsensus(); - CAmount nSum = 0; - for (int nHeight = 0; nHeight < 14000000; nHeight += 1000) - { - CAmount nSubsidy = GetBlockSubsidy(nHeight, consensusParams); - BOOST_CHECK(nSubsidy <= 50 * COIN); - nSum += nSubsidy * 1000; - BOOST_CHECK(MoneyRange(nSum)); - } - BOOST_CHECK_EQUAL(nSum, 2099999997690000ULL); -} - bool ReturnFalse() { return false; } bool ReturnTrue() { return true; } BOOST_AUTO_TEST_CASE(test_combiner_all) diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 46090dc6..c5b68cbd 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -211,7 +211,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) std::vector snapshotOrder; { CTxMemPool::setEntries setAncestorsCalculated; - LOCK(pool.cs); + WRITELOCK(pool.cs); BOOST_CHECK_EQUAL(pool._CalculateMemPoolAncestors(entry.Fee(2000000LL).FromTx(tx7), setAncestorsCalculated, 100, 1000000, 1000, 1000000, dummy), true); diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index d998a27f..40459827 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -62,7 +62,7 @@ class CAddrManCorrupted : public CAddrManSerializationMock CDataStream AddrmanToStream(CAddrManSerializationMock &_addrman) { CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); - ssPeersIn << FLATDATA(Params().DiskMagic()); + ssPeersIn << FLATDATA(pnetMan->getActivePaymentNetwork()->MessageStart()); ssPeersIn << _addrman; std::string str = ssPeersIn.str(); std::vector vchData(str.begin(), str.end()); @@ -113,7 +113,7 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); CAddrMan addrman2; - CAddrDB adb(Params()); + CAddrDB adb; BOOST_CHECK(addrman2.size() == 0); adb.Read(addrman2, ssPeers2); BOOST_CHECK(addrman2.size() == 3); @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(caddrdb_read_corrupted) CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); CAddrMan addrman2; - CAddrDB adb(Params()); + CAddrDB adb; BOOST_CHECK(addrman2.size() == 0); adb.Read(addrman2, ssPeers2); BOOST_CHECK(addrman2.size() == 0); @@ -179,39 +179,4 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode2->fFeeler == false); } -BOOST_AUTO_TEST_CASE(test_getSubVersionEB) -{ - BOOST_CHECK_EQUAL(getSubVersionEB(13800000000), "13800.0"); - BOOST_CHECK_EQUAL(getSubVersionEB(3800000000), "3800.0"); - BOOST_CHECK_EQUAL(getSubVersionEB(14000000), "14.0"); - BOOST_CHECK_EQUAL(getSubVersionEB(1540000), "1.5"); - BOOST_CHECK_EQUAL(getSubVersionEB(1560000), "1.5"); - BOOST_CHECK_EQUAL(getSubVersionEB(210000), "0.2"); - BOOST_CHECK_EQUAL(getSubVersionEB(10000), "0.0"); - BOOST_CHECK_EQUAL(getSubVersionEB(0), "0.0"); -} - -BOOST_AUTO_TEST_CASE(test_userAgentLength) -{ - GlobalConfig config; - - config.SetMaxBlockSize(8000000); - std::string long_uacomment = "very very very very very very very very very " - "very very very very very very very very very " - "very very very very very very very very very " - "very very very very very very very very very " - "very very very very very very very very very " - "very very very very very very very very very " - "very very very very very very very very very " - "very very very very very very long comment"; - gArgs.ForceSetMultiArg("-uacomment", long_uacomment); - - BOOST_CHECK_EQUAL(userAgent(config).size(), MAX_SUBVERSION_LENGTH); - BOOST_CHECK_EQUAL(userAgent(config), "/Bitcoin ABC:0.17.3(EB8.0; very very very very very " - "very very very very very very very very very very very " - "very very very very very very very very very very very " - "very very very very very very very very very very very " - "very very very very very very very ve)/"); -} - BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 51dd2995..dc280c9d 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -107,7 +107,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } std::vector origFeeEst; - std::vector origPriEst; // Highest feerate is 10*baseRate and gets in all blocks, // second highest feerate is 9*baseRate and gets in 9/10 blocks = 90%, // third highest feerate is 8*base rate, and gets in 8/10 blocks = 80%, @@ -120,13 +119,10 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) if (i > 1) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i - 1] <= origFeeEst[i - 2]); - BOOST_CHECK(origPriEst[i - 1] <= origPriEst[i - 2]); } int mult = 11 - i; BOOST_CHECK(origFeeEst[i - 1] < mult * baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i - 1] > mult * baseRate.GetFeePerK() - deltaFee); - BOOST_CHECK(origPriEst[i - 1] < pow(10, mult) * basepri + deltaPri); - BOOST_CHECK(origPriEst[i - 1] > pow(10, mult) * basepri - deltaPri); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 87a556ae..580409ca 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -39,6 +39,7 @@ UniValue CallRPC(std::string args) BOOST_FIXTURE_TEST_SUITE(rpc_tests, TestingSetup) +/* BOOST_AUTO_TEST_CASE(rpc_rawparams) { // Test raw transaction API argument handling @@ -78,8 +79,8 @@ BOOST_AUTO_TEST_CASE(rpc_rawparams) BOOST_CHECK_THROW(CallRPC("signrawtransaction null"), std::runtime_error); BOOST_CHECK_THROW(CallRPC("signrawtransaction ff00"), std::runtime_error); BOOST_CHECK_NO_THROW(CallRPC(std::string("signrawtransaction ") + rawtx)); - BOOST_CHECK_NO_THROW(CallRPC(std::string("signrawtransaction ") + rawtx + " null null NONE|FORKID|ANYONECANPAY")); - BOOST_CHECK_NO_THROW(CallRPC(std::string("signrawtransaction ") + rawtx + " [] [] NONE|FORKID|ANYONECANPAY")); + BOOST_CHECK_NO_THROW(CallRPC(std::string("signrawtransaction ") + rawtx + " null null NONE|ANYONECANPAY")); + BOOST_CHECK_NO_THROW(CallRPC(std::string("signrawtransaction ") + rawtx + " [] [] NONE|ANYONECANPAY")); BOOST_CHECK_THROW(CallRPC(std::string("signrawtransaction ") + rawtx + " null null badenum"), std::runtime_error); // Only check failure cases for sendrawtransaction, there's no network to @@ -90,26 +91,6 @@ BOOST_AUTO_TEST_CASE(rpc_rawparams) BOOST_CHECK_THROW(CallRPC(std::string("sendrawtransaction ") + rawtx + " extra"), std::runtime_error); } -BOOST_AUTO_TEST_CASE(rpc_togglenetwork) -{ - UniValue r; - - r = CallRPC("getnetworkinfo"); - bool netState = find_value(r.get_obj(), "networkactive").get_bool(); - BOOST_CHECK_EQUAL(netState, true); - - r = CallRPC("getnetworkinfo"); - int numConnection = find_value(r.get_obj(), "connections").get_int(); - BOOST_CHECK_EQUAL(numConnection, 0); - - netState = find_value(r.get_obj(), "networkactive").get_bool(); - BOOST_CHECK_EQUAL(netState, false); - - r = CallRPC("getnetworkinfo"); - netState = find_value(r.get_obj(), "networkactive").get_bool(); - BOOST_CHECK_EQUAL(netState, true); -} - BOOST_AUTO_TEST_CASE(rpc_rawsign) { UniValue r; @@ -124,10 +105,10 @@ BOOST_AUTO_TEST_CASE(rpc_rawsign) "\"512103debedc17b3df2badbcdd86d5feb4562b86fe182e5998" "abd8bcd4f122c6155b1b21027e940bb73ab8732bfdf7f9216ece" "fca5b94d6df834e77e108f68e66f126044c052ae\"}]"; - r = CallRPC(std::string("createrawtransaction ") + prevout + " " + "{\"3HqAe9LtNBjnsfM4CyYaWTnvCaUYT7v4oZ\":11}"); + r = CallRPC(std::string("createrawtransaction ") + prevout + " " + "{\"EZTeaB5K4zBE9J2NmptRSdZxcZRZM6ahqe\":11}"); std::string notsigned = r.get_str(); - std::string privkey1 = "\"KzsXybp9jX64P5ekX1KUxRQ79Jht9uzW7LorgwE65i5rWACL6LQe\""; - std::string privkey2 = "\"Kyhdf5LuKTRx4ge69ybABsiUAWjVRK4XGxAKk2FQLp2HjGMy87Z4\""; + std::string privkey1 = "\"QwmG4wQZ3d2tREdYGd8sKGK5dgU1LoFqZia8ShU7XKSF5YocpV2Z\""; + std::string privkey2 = "\"QtxuJ7ZkQzjmCoFNmiJsuu6BMwc3xpWV2ppKaXhq2yo8Tz8DfXTe\""; r = CallRPC(std::string("signrawtransaction ") + notsigned + " " + prevout + " " + "[]"); BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == false); r = CallRPC( @@ -153,7 +134,7 @@ BOOST_AUTO_TEST_CASE(rpc_rawsign_missing_amount) "\"512103debedc17b3df2badbcdd86d5feb4562b86fe182e5998" "abd8bcd4f122c6155b1b21027e940bb73ab8732bfdf7f9216ece" "fca5b94d6df834e77e108f68e66f126044c052ae\"}]"; - r = CallRPC(std::string("createrawtransaction ") + prevout + " " + "{\"3HqAe9LtNBjnsfM4CyYaWTnvCaUYT7v4oZ\":11}"); + r = CallRPC(std::string("createrawtransaction ") + prevout + " " + "{\"EZTeaB5K4zBE9J2NmptRSdZxcZRZM6ahqe\":11}"); std::string notsigned = r.get_str(); std::string privkey1 = "\"KzsXybp9jX64P5ekX1KUxRQ79Jht9uzW7LorgwE65i5rWACL6LQe\""; std::string privkey2 = "\"Kyhdf5LuKTRx4ge69ybABsiUAWjVRK4XGxAKk2FQLp2HjGMy87Z4\""; @@ -174,6 +155,7 @@ BOOST_AUTO_TEST_CASE(rpc_rawsign_missing_amount) BOOST_CHECK(exceptionThrownDueToMissingAmount == true); BOOST_CHECK(errorWasMissingAmount == true); } +*/ BOOST_AUTO_TEST_CASE(rpc_createraw_op_return) { @@ -222,37 +204,37 @@ BOOST_AUTO_TEST_CASE(rpc_createraw_op_return) BOOST_AUTO_TEST_CASE(rpc_format_monetary_values) { - BOOST_CHECK(ValueFromAmount(CAmount(0LL)).write() == "0.00000000"); - BOOST_CHECK(ValueFromAmount(CAmount(1LL)).write() == "0.00000001"); - BOOST_CHECK(ValueFromAmount(CAmount(17622195LL)).write() == "0.17622195"); - BOOST_CHECK(ValueFromAmount(CAmount(50000000LL)).write() == "0.50000000"); - BOOST_CHECK(ValueFromAmount(CAmount(89898989LL)).write() == "0.89898989"); - BOOST_CHECK(ValueFromAmount(CAmount(100000000LL)).write() == "1.00000000"); - BOOST_CHECK(ValueFromAmount(CAmount(2099999999999990LL)).write() == "20999999.99999990"); - BOOST_CHECK(ValueFromAmount(CAmount(2099999999999999LL)).write() == "20999999.99999999"); - - BOOST_CHECK_EQUAL(ValueFromAmount(CAmount(0)).write(), "0.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(123456789 * (COIN / 10000)).write(), "12345.67890000"); - BOOST_CHECK_EQUAL(ValueFromAmount(-1 * COIN).write(), "-1.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(-1 * COIN / 10).write(), "-0.10000000"); - - BOOST_CHECK_EQUAL(ValueFromAmount(100000000 * COIN).write(), "100000000.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(10000000 * COIN).write(), "10000000.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(1000000 * COIN).write(), "1000000.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(100000 * COIN).write(), "100000.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(10000 * COIN).write(), "10000.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(1000 * COIN).write(), "1000.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(100 * COIN).write(), "100.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(10 * COIN).write(), "10.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN).write(), "1.00000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 10).write(), "0.10000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 100).write(), "0.01000000"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 1000).write(), "0.00100000"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 10000).write(), "0.00010000"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 100000).write(), "0.00001000"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 1000000).write(), "0.00000100"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 10000000).write(), "0.00000010"); - BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 100000000).write(), "0.00000001"); + BOOST_CHECK(ValueFromAmount(CAmount(0LL)).write() == "0.000000"); + BOOST_CHECK(ValueFromAmount(CAmount(1LL)).write() == "0.000001"); + BOOST_CHECK(ValueFromAmount(CAmount(17622195LL)).write() == "17.622195"); + BOOST_CHECK(ValueFromAmount(CAmount(50000000LL)).write() == "50.000000"); + BOOST_CHECK(ValueFromAmount(CAmount(89898989LL)).write() == "89.898989"); + BOOST_CHECK(ValueFromAmount(CAmount(100000000LL)).write() == "100.000000"); + BOOST_CHECK(ValueFromAmount(CAmount(2099999999999990LL)).write() == "2099999999.999990"); + BOOST_CHECK(ValueFromAmount(CAmount(2099999999999999LL)).write() == "2099999999.999999"); + + BOOST_CHECK_EQUAL(ValueFromAmount(CAmount(0)).write(), "0.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(123456789 * (COIN / 10000)).write(), "12345.678900"); + BOOST_CHECK_EQUAL(ValueFromAmount(-1 * COIN).write(), "-1.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(-1 * COIN / 10).write(), "-0.100000"); + + BOOST_CHECK_EQUAL(ValueFromAmount(100000000 * COIN).write(), "100000000.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(10000000 * COIN).write(), "10000000.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(1000000 * COIN).write(), "1000000.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(100000 * COIN).write(), "100000.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(10000 * COIN).write(), "10000.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(1000 * COIN).write(), "1000.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(100 * COIN).write(), "100.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(10 * COIN).write(), "10.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN).write(), "1.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 10).write(), "0.100000"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 100).write(), "0.010000"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 1000).write(), "0.001000"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 10000).write(), "0.000100"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 100000).write(), "0.000010"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 1000000).write(), "0.000001"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 10000000).write(), "0.000000"); + BOOST_CHECK_EQUAL(ValueFromAmount(COIN / 100000000).write(), "0.000000"); } static UniValue ValueFromString(const std::string &str) @@ -264,53 +246,52 @@ static UniValue ValueFromString(const std::string &str) BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) { - BOOST_CHECK_THROW(AmountFromValue(ValueFromString("-0.00000001")), UniValue); + BOOST_CHECK_THROW(AmountFromValue(ValueFromString("-0.000001")), UniValue); BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0")), CAmount(0LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000000")), CAmount(0LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000001")), CAmount(1LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.17622195")), CAmount(17622195LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.5")), CAmount(50000000LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.50000000")), CAmount(50000000LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.89898989")), CAmount(89898989LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("1.00000000")), CAmount(100000000LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("20999999.9999999")), CAmount(2099999999999990LL)); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("20999999.99999999")), CAmount(2099999999999999LL)); - - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("1e-8")), COIN / 100000000); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.1e-7")), COIN / 100000000); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.01e-6")), COIN / 100000000); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0." - "0000000000000000000000000000000000000000000000000000" - "000000000000000000000001e+68")), - COIN / 100000000); - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("10000000000000000000000000000000000000" - "000000000000000000000000000e-64")), + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.000000")), CAmount(0LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.000001")), CAmount(1LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.176221")), CAmount(176221LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.5")), CAmount(500000LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("50.000000")), CAmount(50000000LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("89.898989")), CAmount(89898989LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("100.000000")), CAmount(100000000LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("2099999999.99999")), CAmount(2099999999999990LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("2099999999.999999")), CAmount(2099999999999999LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("1e-6")), SATOSHI); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.1e-5")), SATOSHI); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.01e-4")), SATOSHI); + BOOST_CHECK_EQUAL(AmountFromValue_Original(ValueFromString("0.00000" + "00000000000000000000000000000000000000000000000000" + "0000000000001e+62")), + SATOSHI); + BOOST_CHECK_EQUAL(AmountFromValue_Original(ValueFromString("10000000000000000000000000000000000000" + "0000000000000000000000000e-62")), COIN); BOOST_CHECK_EQUAL( - AmountFromValue(ValueFromString("0." - "000000000000000000000000000000000000000000000000000000000000000100" - "000000000000000000000000000000000000000000000000000e64")), + AmountFromValue_Original(ValueFromString("0." + "000000000000000000000000000000000000000000000000000000000000010000" + "00000000000000000000000000000000000000000000000000000e62")), COIN); // should fail - BOOST_CHECK_THROW(AmountFromValue(ValueFromString("1e-9")), UniValue); + BOOST_CHECK_THROW(AmountFromValue_Original(ValueFromString("1e-9")), UniValue); // should fail - BOOST_CHECK_THROW(AmountFromValue(ValueFromString("0.000000019")), UniValue); + BOOST_CHECK_THROW(AmountFromValue_Original(ValueFromString("0.000000019")), UniValue); // should pass, cut trailing 0 - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000001000000")), CAmount(1LL)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000100000000")), CAmount(1LL)); // should fail - BOOST_CHECK_THROW(AmountFromValue(ValueFromString("19e-9")), UniValue); + BOOST_CHECK_THROW(AmountFromValue_Original(ValueFromString("19e-9")), UniValue); // should pass, leading 0 is present - BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.19e-6")), CAmount(19)); + BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.19e-4")), CAmount(19)); // overflow error - BOOST_CHECK_THROW(AmountFromValue(ValueFromString("92233720368.54775808")), UniValue); + BOOST_CHECK_THROW(AmountFromValue_Original(ValueFromString("92233720368.54775808")), UniValue); // overflow error - BOOST_CHECK_THROW(AmountFromValue(ValueFromString("1e+11")), UniValue); + BOOST_CHECK_THROW(AmountFromValue_Original(ValueFromString("1e+11")), UniValue); // overflow error signless - BOOST_CHECK_THROW(AmountFromValue(ValueFromString("1e11")), UniValue); + BOOST_CHECK_THROW(AmountFromValue_Original(ValueFromString("1e11")), UniValue); // overflow error - BOOST_CHECK_THROW(AmountFromValue(ValueFromString("93e+9")), UniValue); + BOOST_CHECK_THROW(AmountFromValue_Original(ValueFromString("93e+9")), UniValue); } BOOST_AUTO_TEST_CASE(json_parse_errors) @@ -324,7 +305,7 @@ BOOST_AUTO_TEST_CASE(json_parse_errors) // should fail, missing leading 0, therefore invalid JSON BOOST_CHECK_THROW(AmountFromValue(ParseNonRFCJSONValue(".19e-6")), std::runtime_error); BOOST_CHECK_EQUAL( - AmountFromValue(ParseNonRFCJSONValue("0.00000000000000000000000000000000000001e+30 ")), CAmount(1)); + AmountFromValue(ParseNonRFCJSONValue("0.00000000000000000000000000000000000001e+30 ")), CAmount(0)); // Invalid, initial garbage BOOST_CHECK_THROW(ParseNonRFCJSONValue("[1.0"), std::runtime_error); BOOST_CHECK_THROW(ParseNonRFCJSONValue("a1.0"), std::runtime_error); diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index 508f07c8..88585d4d 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -108,20 +108,26 @@ BOOST_AUTO_TEST_CASE(sign) // All of the above should be OK, and the txTos have valid signatures // Check to make sure signature verification fails if we use the wrong ScriptSig: for (int i = 0; i < 8; i++) + { for (int j = 0; j < 8; j++) { CScript sigSave = txTo[i].vin[0].scriptSig; txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; const CTxOut &output = txFrom.vout[txTo[i].vin[0].prevout.n]; - bool sigOK = CScriptCheck(output.scriptPubKey, output.nValue, txTo[i], 0, - SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false)(); + bool sigOK = CScriptCheck( + output.scriptPubKey, output.nValue, txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false)(); if (i == j) + { BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); + } else + { BOOST_CHECK_MESSAGE(!sigOK, strprintf("VerifySignature %d %d", i, j)); + } txTo[i].vin[0].scriptSig = sigSave; } + } } BOOST_AUTO_TEST_CASE(norecurse) diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 592a95e4..1a1e4103 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -42,6 +42,8 @@ BasicTestingSetup::BasicTestingSetup(const std::string &chainName) pnetMan = new CNetworkManager(); pwallet = new CWallet("walletFile"); pnetMan->SetParams(chainName); + // Deterministic randomness for tests. + g_connman = std::make_unique(0x1337, 0x1337); } BasicTestingSetup::~BasicTestingSetup() { ECC_Stop(); } @@ -71,6 +73,7 @@ TestingSetup::~TestingSetup() pnetMan->getChainActive()->pcoinsTip.reset(); pcoinsdbview = nullptr; pnetMan->getChainActive()->pblocktree.reset(); + g_connman.reset(); fs::remove_all(pathTemp); } diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index b0bdb259..2cc0f360 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -10,6 +10,8 @@ #include +class CConnman; +class CNode; extern CNetworkManager *pnetman; diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp deleted file mode 100644 index 7a643e77..00000000 --- a/src/test/versionbits_tests.cpp +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright (c) 2014-2015 The Bitcoin Core developers -// Copyright (c) 2015-2017 The Bitcoin Unlimited developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "versionbits.h" -#include "chain/chain.h" - -#include "consensus/params.h" -#include "main.h" -#include "random.h" -#include "test/test_bitcoin.h" - -#include - -/* Define a virtual block time, one block per 10 minutes after Nov 14 2014, 0:55:36am */ -int32_t TestTime(int nHeight) { return 1415926536 + 600 * nHeight; } -static const Consensus::Params paramsDummy = Consensus::Params(); - -class TestConditionChecker : public AbstractThresholdConditionChecker -{ -private: - mutable ThresholdConditionCache cache; - -public: - int64_t BeginTime(const Consensus::Params ¶ms) const { return TestTime(10000); } - int64_t EndTime(const Consensus::Params ¶ms) const { return TestTime(20000); } - int Period(const Consensus::Params ¶ms) const { return 1000; } - int Threshold(const Consensus::Params ¶ms) const { return 900; } - bool Condition(const CBlockIndex *pindex, const Consensus::Params ¶ms) const - { - return (pindex->nVersion & 0x100); - } - - ThresholdState GetStateFor(const CBlockIndex *pindexPrev) const - { - return AbstractThresholdConditionChecker::GetStateFor(pindexPrev, paramsDummy, cache); - } -}; - -#define CHECKERS 6 - -class VersionBitsTester -{ - // A fake blockchain - std::vector vpblock; - - // 6 independent checkers for the same bit. - // The first one performs all checks, the second only 50%, the third only 25%, etc... - // This is to test whether lack of cached information leads to the same results. - TestConditionChecker checker[CHECKERS]; - - // Test counter (to identify failures) - int num; - -public: - VersionBitsTester() : num(0) {} - VersionBitsTester &Reset() - { - for (unsigned int i = 0; i < vpblock.size(); i++) - { - delete vpblock[i]; - } - for (unsigned int i = 0; i < CHECKERS; i++) - { - checker[i] = TestConditionChecker(); - } - vpblock.clear(); - return *this; - } - - ~VersionBitsTester() { Reset(); } - VersionBitsTester &Mine(unsigned int height, int32_t nTime, int32_t nVersion) - { - while (vpblock.size() < height) - { - CBlockIndex *pindex = new CBlockIndex(); - pindex->nHeight = vpblock.size(); - pindex->pprev = vpblock.size() > 0 ? vpblock.back() : NULL; - pindex->nTime = nTime; - pindex->nVersion = nVersion; - pindex->BuildSkip(); - vpblock.push_back(pindex); - } - return *this; - } - - VersionBitsTester &TestDefined() - { - for (int i = 0; i < CHECKERS; i++) - { - if ((insecure_rand() & ((1 << i) - 1)) == 0) - { - BOOST_CHECK_MESSAGE( - checker[i].GetStateFor(vpblock.empty() ? NULL : vpblock.back()) == THRESHOLD_DEFINED, - strprintf("Test %i for DEFINED", num)); - } - } - num++; - return *this; - } - - VersionBitsTester &TestStarted() - { - for (int i = 0; i < CHECKERS; i++) - { - if ((insecure_rand() & ((1 << i) - 1)) == 0) - { - BOOST_CHECK_MESSAGE( - checker[i].GetStateFor(vpblock.empty() ? NULL : vpblock.back()) == THRESHOLD_STARTED, - strprintf("Test %i for STARTED", num)); - } - } - num++; - return *this; - } - - VersionBitsTester &TestLockedIn() - { - for (int i = 0; i < CHECKERS; i++) - { - if ((insecure_rand() & ((1 << i) - 1)) == 0) - { - BOOST_CHECK_MESSAGE( - checker[i].GetStateFor(vpblock.empty() ? NULL : vpblock.back()) == THRESHOLD_LOCKED_IN, - strprintf("Test %i for LOCKED_IN", num)); - } - } - num++; - return *this; - } - - VersionBitsTester &TestActive() - { - for (int i = 0; i < CHECKERS; i++) - { - if ((insecure_rand() & ((1 << i) - 1)) == 0) - { - BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? NULL : vpblock.back()) == THRESHOLD_ACTIVE, - strprintf("Test %i for ACTIVE", num)); - } - } - num++; - return *this; - } - - VersionBitsTester &TestFailed() - { - for (int i = 0; i < CHECKERS; i++) - { - if ((insecure_rand() & ((1 << i) - 1)) == 0) - { - BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? NULL : vpblock.back()) == THRESHOLD_FAILED, - strprintf("Test %i for FAILED", num)); - } - } - num++; - return *this; - } - - CBlockIndex *Tip() { return vpblock.size() ? vpblock.back() : NULL; } -}; - -BOOST_FIXTURE_TEST_SUITE(versionbits_tests, TestingSetup) - -BOOST_AUTO_TEST_CASE(versionbits_test) -{ - for (int i = 0; i < 64; i++) - { - // DEFINED -> FAILED - VersionBitsTester() - .TestDefined() - .Mine(1, TestTime(1), 0x100) - .TestDefined() - .Mine(11, TestTime(11), 0x100) - .TestDefined() - .Mine(989, TestTime(989), 0x100) - .TestDefined() - .Mine(999, TestTime(20000), 0x100) - .TestDefined() - .Mine(1000, TestTime(20000), 0x100) - .TestFailed() - .Mine(1999, TestTime(30001), 0x100) - .TestFailed() - .Mine(2000, TestTime(30002), 0x100) - .TestFailed() - .Mine(2001, TestTime(30003), 0x100) - .TestFailed() - .Mine(2999, TestTime(30004), 0x100) - .TestFailed() - .Mine(3000, TestTime(30005), 0x100) - .TestFailed() - - // DEFINED -> STARTED -> FAILED - .Reset() - .TestDefined() - .Mine(1, TestTime(1), 0) - .TestDefined() - // One second more and it would be defined - .Mine(1000, TestTime(10000) - 1, 0x100) - .TestDefined() - .Mine(2000, TestTime(10000), 0x100) - .TestStarted() // So that's what happens the next period - .Mine(2051, TestTime(10010), 0) - .TestStarted() // 51 old blocks - .Mine(2950, TestTime(10020), 0x100) - .TestStarted() // 899 new blocks - .Mine(3000, TestTime(20000), 0) - .TestFailed() // 50 old blocks (so 899 out of the past 1000) - .Mine(4000, TestTime(20010), 0x100) - .TestFailed() - - // DEFINED -> STARTED -> FAILED while threshold reached - .Reset() - .TestDefined() - .Mine(1, TestTime(1), 0) - .TestDefined() - // One second more and it would be defined - .Mine(1000, TestTime(10000) - 1, 0x101) - .TestDefined() - .Mine(2000, TestTime(10000), 0x101) - .TestStarted() // So that's what happens the next period - .Mine(2999, TestTime(30000), 0x100) - .TestStarted() // 999 new blocks - // 1 new block (so 1000 out of the past 1000 are new) - .Mine(3000, TestTime(30000), 0x100) - .TestFailed() - .Mine(3999, TestTime(30001), 0) - .TestFailed() - .Mine(4000, TestTime(30002), 0) - .TestFailed() - .Mine(14333, TestTime(30003), 0) - .TestFailed() - .Mine(24000, TestTime(40000), 0) - .TestFailed() - - // DEFINED -> STARTED -> LOCKEDIN at the last minute -> ACTIVE - .Reset() - .TestDefined() - .Mine(1, TestTime(1), 0) - .TestDefined() - // One second more and it would be defined - .Mine(1000, TestTime(10000) - 1, 0x101) - .TestDefined() - .Mine(2000, TestTime(10000), 0x101) - .TestStarted() // So that's what happens the next period - .Mine(2050, TestTime(10010), 0x200) - .TestStarted() // 50 old blocks - .Mine(2950, TestTime(10020), 0x100) - .TestStarted() // 900 new blocks - .Mine(2999, TestTime(19999), 0x200) - .TestStarted() // 49 old blocks - // 1 old block (so 900 out of the past 1000) - .Mine(3000, TestTime(29999), 0x200) - .TestLockedIn() - .Mine(3999, TestTime(30001), 0) - .TestLockedIn() - .Mine(4000, TestTime(30002), 0) - .TestActive() - .Mine(14333, TestTime(30003), 0) - .TestActive() - .Mine(24000, TestTime(40000), 0) - .TestActive(); - } - - // Sanity checks of version bit deployments - for (int i = 0; i < (int)MAX_VERSION_BITS_DEPLOYMENTS; i++) - { - uint32_t bitmask = VersionBitsMask(mainnetParams, (DeploymentPos)i); - // Make sure that no deployment tries to set an invalid bit. - BOOST_CHECK_EQUAL(bitmask & ~(uint32_t)VERSIONBITS_TOP_MASK, bitmask); - - // Verify that the deployment windows of different deployment using the - // same bit are disjoint. - // This test may need modification at such time as a new deployment - // is proposed that reuses the bit of an activated soft fork, before the - // end time of that soft fork. (Alternatively, the end time of that - // activated soft fork could be later changed to be earlier to avoid - // overlap.) - for (int j = i + 1; j < (int)MAX_VERSION_BITS_DEPLOYMENTS; j++) - { - if (VersionBitsMask(mainnetParams, (DeploymentPos)j) == bitmask) - { - BOOST_CHECK(mainnetParams.vDeployments[j].nStartTime > mainnetParams.vDeployments[i].nTimeout || - mainnetParams.vDeployments[i].nStartTime > mainnetParams.vDeployments[j].nTimeout); - } - } - } -} - -BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) -{ - // Check that ComputeBlockVersion will set the appropriate bit correctly - // on mainnet. - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); - - // Use the TESTDUMMY deployment for testing purposes. - int64_t bit = mainnetParams.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].bit; - int64_t nStartTime = mainnetParams.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nStartTime; - int64_t nTimeout = mainnetParams.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout; - - assert(nStartTime < nTimeout); - - // In the first chain, test that the bit is set by CBV until it has failed. - // In the second chain, test the bit is set by CBV while STARTED and - // LOCKED-IN, and then no longer set while ACTIVE. - VersionBitsTester firstChain, secondChain; - - // Start generating blocks before nStartTime - int64_t nTime = nStartTime - 1; - - // Before MedianTimePast of the chain has crossed nStartTime, the bit - // should not be set. - CBlockIndex *lastBlock = NULL; - lastBlock = firstChain.Mine(2016, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit), 0); - - // Mine 2011 more blocks at the old time, and check that CBV isn't setting the bit yet. - for (int i = 1; i < 2012; i++) - { - lastBlock = firstChain.Mine(2016 + i, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - // This works because VERSIONBITS_LAST_OLD_BLOCK_VERSION happens - // to be 4, and the bit we're testing happens to be bit 28. - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit), 0); - } - // Now mine 5 more blocks at the start time -- MTP should not have passed yet, so - // CBV should still not yet set the bit. - nTime = nStartTime; - for (int i = 2012; i <= 2016; i++) - { - lastBlock = firstChain.Mine(2016 + i, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit), 0); - } - - // Advance to the next period and transition to STARTED, - lastBlock = firstChain.Mine(6048, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - // so ComputeBlockVersion should now set the bit, - BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit)) != 0); - // and should also be using the VERSIONBITS_TOP_BITS. - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & VERSIONBITS_TOP_MASK, VERSIONBITS_TOP_BITS); - - // Check that ComputeBlockVersion will set the bit until nTimeout - nTime += 600; - int blocksToMine = 4032; // test blocks for up to 2 time periods - int nHeight = 6048; - // These blocks are all before nTimeout is reached. - while (nTime < nTimeout && blocksToMine > 0) - { - lastBlock = firstChain.Mine(nHeight + 1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit)) != 0); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & VERSIONBITS_TOP_MASK, VERSIONBITS_TOP_BITS); - blocksToMine--; - nTime += 600; - nHeight += 1; - }; - - nTime = nTimeout; - // FAILED is only triggered at the end of a period, so CBV should be setting - // the bit until the period transition. - for (int i = 0; i < 2015; i++) - { - lastBlock = firstChain.Mine(nHeight + 1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit)) != 0); - nHeight += 1; - } - // The next block should trigger no longer setting the bit. - lastBlock = firstChain.Mine(nHeight + 1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit), 0); - - // On a new chain: - // verify that the bit will be set after lock-in, and then stop being set - // after activation. - nTime = nStartTime; - - // Mine one period worth of blocks, and check that the bit will be on for the - // next period. - lastBlock = secondChain.Mine(2016, nStartTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit)) != 0); - - // Mine another period worth of blocks, signaling the new bit. - lastBlock = secondChain.Mine(4032, nStartTime, VERSIONBITS_TOP_BITS | (1 << bit)).Tip(); - // After one period of setting the bit on each block, it should have locked in. - // We keep setting the bit for one more period though, until activation. - BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit)) != 0); - - // Now check that we keep mining the block until the end of this period, and - // then stop at the beginning of the next period. - lastBlock = secondChain.Mine(6047, nStartTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit)) != 0); - lastBlock = secondChain.Mine(6048, nStartTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit), 0); - - // Finally, verify that after a soft fork has activated, CBV no longer uses - // VERSIONBITS_LAST_OLD_BLOCK_VERSION. - // BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & VERSIONBITS_TOP_MASK, VERSIONBITS_TOP_BITS); -} - - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.cpp b/src/txmempool.cpp index e538b908..b2095518 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1355,8 +1355,10 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector *pvNoSpends } if (maxFeeRateRemoved > CFeeRate(0)) - LogPrintf( + { + LogPrint( "MEMPOOL", "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + } } void CTxMemPool::UpdateTransactionsPerSecond() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index e7968560..4f0091ef 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1330,7 +1330,14 @@ void RelayTransaction(const CTransaction &tx, CConnman &connman); bool CWalletTx::RelayWalletTransaction(CConnman *connman) { - assert(pwallet->GetBroadcastTransactions()); + if (pwallet == nullptr) + { + return false; + } + if (!pwallet->GetBroadcastTransactions()) + { + return false; + } if (tx->IsCoinBase() || tx->IsCoinStake() || isAbandoned() || GetDepthInMainChain() != 0) { return false; @@ -1578,20 +1585,22 @@ std::vector CWallet::ResendWalletTransactionsBefore(int64_t nTime, CCon { std::vector result; - LOCK(cs_wallet); - // Sort them in chronological order - std::multimap mapSorted; - for (auto const &item : mapWallet) + std::multimap mapSorted; { - CWalletTx wtx = item.second; - // Don't rebroadcast if newer than nTime: - if (wtx.nTimeReceived > nTime) - continue; - mapSorted.insert(std::make_pair(wtx.nTimeReceived, &wtx)); + LOCK(cs_wallet); + // Sort them in chronological order + for (auto &item : mapWallet) + { + CWalletTx &wtx = item.second; + // Don't rebroadcast if newer than nTime: + if (wtx.nTimeReceived > nTime) + continue; + mapSorted.insert(std::make_pair(wtx.nTimeReceived, wtx)); + } } - for (auto const &item : mapSorted) + for (auto &item : mapSorted) { - CWalletTx &wtx = *item.second; + CWalletTx &wtx = item.second; if (wtx.RelayWalletTransaction(connman)) result.push_back(wtx.tx->GetHash()); } @@ -1720,7 +1729,10 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const return nTotal; } -void CWallet::AvailableCoins(std::vector &vCoins, bool fOnlyConfirmed, bool fIncludeZeroValue) const +void CWallet::AvailableCoins(std::vector &vCoins, + bool fOnlyConfirmed, + bool fIncludeZeroValue, + const std::vector &vin) const { vCoins.clear(); @@ -1751,6 +1763,32 @@ void CWallet::AvailableCoins(std::vector &vCoins, bool fOnlyConfirmed, for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { + if (vin.empty() == false) + { + bool found = false; + bool input = false; + for (const auto &iter : vin) + { + if (iter.prevout.hash == pcoin->tx->GetHash()) + { + if (i == iter.prevout.n) + { + input = true; + break; + } + found = true; + } + } + if (found == false && input == false) + { + break; + } + if (found == true && input == false) + { + continue; + } + } + isminetype mine = IsMine(pcoin->tx->vout[i]); if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && !IsLockedCoin((*it).first, i) && (pcoin->tx->vout[i].nValue > 0 || fIncludeZeroValue)) @@ -1954,68 +1992,25 @@ bool CWallet::SelectCoinsMinConf(const CAmount &nTargetValue, return true; } -void CWallet::AvailableCoinsByOwner(std::vector &vCoins, const CRecipient &recipient) const +bool CWallet::SelectCoins(const CAmount &nTargetValue, + std::set > &setCoinsRet, + CAmount &nValueRet, + const std::vector &vin) const { - vCoins.clear(); - CTxDestination owneraddr; - if (!ExtractDestination(recipient.scriptPubKey, owneraddr)) - { - return; - } + std::vector vCoins; + AvailableCoins(vCoins, true, false, vin); + if (vin.empty() == false) { - LOCK2(cs_main, cs_wallet); - for (std::map::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const COutput &out : vCoins) { - const uint256 &wtxid = it->first; - const CWalletTx *pcoin = &(*it).second; - - if (!CheckFinalTx(*(pcoin->tx))) - continue; - - if (!pcoin->IsTrusted()) + if (!out.fSpendable) continue; - - if ((pcoin->tx->IsCoinBase() || pcoin->tx->IsCoinStake()) && pcoin->GetBlocksToMaturity() > 0) - continue; - - int nDepth = pcoin->GetDepthInMainChain(); - if (nDepth < 0) - continue; - - // We should not consider coins which aren't at least in our mempool - // It's possible for these to be conflicted via ancestors which we may never be able to detect - if (nDepth == 0 && !pcoin->InMempool()) - continue; - - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) - { - CTxDestination addr; - if (!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, addr)) - { - continue; - } - if (addr == owneraddr) - { - isminetype mine = IsMine(pcoin->tx->vout[i]); - if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && !IsLockedCoin((*it).first, i) && - (pcoin->tx->vout[i].nValue > 0)) - { - vCoins.push_back(COutput(pcoin, i, nDepth, - ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || ((mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO))); - } - } - } + nValueRet += out.tx->tx->vout[out.i].nValue; + setCoinsRet.insert(std::make_pair(out.tx, out.i)); } + return (nValueRet >= nTargetValue); } -} - -bool CWallet::SelectCoins(const CAmount &nTargetValue, - std::set > &setCoinsRet, - CAmount &nValueRet) const -{ - std::vector vCoins; - AvailableCoins(vCoins, true); // calculate value from preset inputs and store them std::set > setPresetCoins; @@ -2070,7 +2065,7 @@ bool CWallet::FundTransaction(CTransaction &tx, CReserveKey reservekey(this); CWalletTx wtx; - if (!CreateTransaction(vecSend, wtx, reservekey, nFeeRet, nChangePosRet, strFailReason, false)) + if (!CreateTransaction(vecSend, wtx, reservekey, nFeeRet, nChangePosRet, strFailReason, false, tx.vin)) return false; if (nChangePosRet != -1) @@ -2103,7 +2098,8 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, CAmount &nFeeRet, int &nChangePosRet, std::string &strFailReason, - bool sign) + bool sign, + const std::vector &vin) { if (fWalletUnlockStakingOnly) { @@ -2219,7 +2215,7 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, // Choose coins to use std::set > setCoins; CAmount nValueIn = 0; - if (!SelectCoins(nValueToSelect, setCoins, nValueIn)) + if (!SelectCoins(nValueToSelect, setCoins, nValueIn, vin)) { strFailReason = ("Insufficient funds"); return false; @@ -2377,9 +2373,9 @@ bool CWallet::CreateTransaction(const std::vector &vecSend, // Remove scriptSigs if we used dummy signatures for fee calculation if (!sign) { - for (auto &vin : txNew.vin) + for (auto &_vin : txNew.vin) { - vin.scriptSig = CScript(); + _vin.scriptSig = CScript(); } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index d6e2ce71..ef495f34 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -318,6 +318,8 @@ class CWalletTx : public CMerkleTx nImmatureWatchCreditCached = 0; nChangeCached = 0; nOrderPos = -1; + hashBlock = uint256(); + nIndex = -1; } ADD_SERIALIZE_METHODS @@ -485,7 +487,8 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface CAmount &nValueRet) const; bool SelectCoins(const CAmount &nTargetValue, std::set > &setCoinsRet, - CAmount &nValueRet) const; + CAmount &nValueRet, + const std::vector &vin) const; bool SelectCoins(CAmount nTargetValue, unsigned int nSpendTime, std::set > &setCoinsRet, @@ -597,8 +600,10 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface /** * populate vCoins with vector of available COutputs. */ - void AvailableCoinsByOwner(std::vector &vCoins, const CRecipient &recipient) const; - void AvailableCoins(std::vector &vCoins, bool fOnlyConfirmed = true, bool fIncludeZeroValue = false) const; + void AvailableCoins(std::vector &vCoins, + bool fOnlyConfirmed = true, + bool fIncludeZeroValue = false, + const std::vector &vin = {}) const; /** * Shuffle and select coins until nTargetValue is reached while avoiding @@ -720,7 +725,8 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface CAmount &nFeeRet, int &nChangePosRet, std::string &strFailReason, - bool sign = true); + bool sign = true, + const std::vector &vin = {}); bool CommitTransaction(CWalletTx &wtxNew, CReserveKey &reservekey, CConnman *connman, CValidationState &state); From da1ad9acc43755c636e5970b98f1aa359e5747fc Mon Sep 17 00:00:00 2001 From: Griffith Date: Sun, 28 Apr 2019 02:09:35 +0900 Subject: [PATCH 39/46] gitian build on docker in ubuntu18 container (#161) * gitian build on docker in ubuntu18 container * remove rsm from dist subdirs * add missing rsm test header * update biplist package for osx * fix incorrect header in rsm tests --- configure.ac | 3 + contrib/gitian-descriptors/gitian-arm.yml | 14 +-- contrib/gitian-descriptors/gitian-linux.yml | 8 +- contrib/gitian-descriptors/gitian-osx.yml | 4 +- contrib/gitian-descriptors/gitian-win.yml | 11 +- depends/packages/bdb.mk | 2 +- depends/packages/boost.mk | 2 +- depends/packages/native_biplist.mk | 6 +- doc/gitian-building.md | 121 ++++---------------- src/Makefile.am | 2 +- src/Makefile.test.include | 3 +- src/compat/glibc_compat.cpp | 76 +++++++++--- src/rsm/test/rsm_promotion_tests.cpp | 2 +- src/rsm/test/rsm_simple_tests.cpp | 2 +- src/rsm/test/rsm_starvation_tests.cpp | 2 +- 15 files changed, 113 insertions(+), 145 deletions(-) diff --git a/configure.ac b/configure.ac index 0f7fe7b8..e8fc55a8 100644 --- a/configure.ac +++ b/configure.ac @@ -549,6 +549,9 @@ if test x$use_glibc_compat != xno; then [ fdelt_type="long int"]) AC_MSG_RESULT($fdelt_type) AC_DEFINE_UNQUOTED(FDELT_TYPE, $fdelt_type,[parameter and return value type for __fdelt_chk]) + AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) + AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) + AX_CHECK_LINK_FLAG([[-Wl,--wrap=logf]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=logf"]) else AC_SEARCH_LIBS([clock_gettime],[rt]) fi diff --git a/contrib/gitian-descriptors/gitian-arm.yml b/contrib/gitian-descriptors/gitian-arm.yml index 01510a95..e4f4ec67 100644 --- a/contrib/gitian-descriptors/gitian-arm.yml +++ b/contrib/gitian-descriptors/gitian-arm.yml @@ -2,21 +2,21 @@ name: "eccoin-linux-0.2.5.12" enable_cache: true suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: - "curl" - "g++-aarch64-linux-gnu" -- "g++-4.8-aarch64-linux-gnu" -- "gcc-4.8-aarch64-linux-gnu" +- "g++-7-aarch64-linux-gnu" +- "gcc-7-aarch64-linux-gnu" - "binutils-aarch64-linux-gnu" - "g++-arm-linux-gnueabihf" -- "g++-4.8-arm-linux-gnueabihf" -- "gcc-4.8-arm-linux-gnueabihf" +- "g++-7-arm-linux-gnueabihf" +- "gcc-7-arm-linux-gnueabihf" - "binutils-arm-linux-gnueabihf" -- "g++-4.8-multilib" -- "gcc-4.8-multilib" +- "g++-7-multilib" +- "gcc-7-multilib" - "binutils-gold" - "git-core" - "pkg-config" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 06763ec5..1437195d 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -2,16 +2,16 @@ name: "eccoin-linux-0.2.5.12" enable_cache: true suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: - "curl" - "binutils" -- "g++-4.8-multilib" -- "gcc-4.8-multilib" +- "g++-7-multilib" +- "gcc-7-multilib" - "binutils-gold" -- "git-core" +- "git" - "pkg-config" - "autoconf" - "libtool" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 422ce899..41ff94c3 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -2,13 +2,13 @@ name: "eccoin-osx-0.2.5.12" enable_cache: true suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: - "curl" - "g++" -- "git-core" +- "git" - "pkg-config" - "autoconf" - "librsvg2-bin" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 1c6b0120..3661764d 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -2,13 +2,13 @@ name: "eccoin-win-0.2.5.12" enable_cache: true suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: - "curl" - "g++" -- "git-core" +- "git" - "pkg-config" - "autoconf" - "libtool" @@ -21,6 +21,7 @@ packages: - "zip" - "ca-certificates" - "python" +- "rename" reference_datetime: "2018-10-12 00:00:00" remotes: - "url": "https://github.com/project-ecc/eccoin.git" @@ -29,8 +30,8 @@ files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-gui-tests" - FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" + FAKETIME_HOST_PROGS="ar ranlib nm windres strip" FAKETIME_PROGS="date makensis zip" export QT_RCC_TEST=1 @@ -81,7 +82,7 @@ script: | done for prog in gcc g++; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index e2b73b62..1ff4cc37 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -9,7 +9,7 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication $(package)_config_opts_mingw32=--enable-mingw $(package)_config_opts_linux=--with-pic -$(package)_cxxflags=-std=c++11 +$(package)_cxxflags=-std=c++14 endef define $(package)_preprocess_cmds diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index bf773ccd..4e1008a5 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -20,7 +20,7 @@ $(package)_archiver_$(host_os)=$($(package)_ar) $(package)_toolset_darwin=darwin $(package)_archiver_darwin=$($(package)_libtool) $(package)_config_libraries=chrono,filesystem,program_options,system,thread,test -$(package)_cxxflags=-std=c++11 -fvisibility=hidden +$(package)_cxxflags=-std=c++14 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC endef diff --git a/depends/packages/native_biplist.mk b/depends/packages/native_biplist.mk index eb8672d5..5f247e9b 100644 --- a/depends/packages/native_biplist.mk +++ b/depends/packages/native_biplist.mk @@ -1,8 +1,8 @@ package=native_biplist -$(package)_version=0.9 -$(package)_download_path=https://pypi.python.org/packages/source/b/biplist +$(package)_version=1.0.3 +$(package)_download_path=https://bitbucket.org/wooster/biplist/downloads $(package)_file_name=biplist-$($(package)_version).tar.gz -$(package)_sha256_hash=b57cadfd26e4754efdf89e9e37de87885f9b5c847b2615688ca04adfaf6ca604 +$(package)_sha256_hash=4c0549764c5fe50b28042ec21aa2e14fe1a2224e239a1dae77d9e7f3932aa4c6 $(package)_install_libdir=$(build_prefix)/lib/python/dist-packages define $(package)_build_cmds diff --git a/doc/gitian-building.md b/doc/gitian-building.md index c3977247..9457b011 100644 --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -1,109 +1,34 @@ -# Building Eccoin with Gitian +Gitian build +============ -Note: These instructions were created on a machine that natively ran Ubuntu 16.04.5 (Xenial). No virtualization was used. +This is guide take for granted that you are using Ubuntu bionic 18.04 as host OS. the aim of the document is to be able to produce deterministic binaries using gitian-tools and docker containers. -## Verify hardware compatability -Verify that your machine has the hardware virtualizations extensions to build -#### For intel the following command should give you a list of flags +Prerequisite +------------- -`grep -e 'vmx' /proc/cpuinfo` +These are steps that as to be executed once and that don't need to be repeated for every new gitian build process. -#### For amd the following command should give you a list of flags - -`grep -e 'svm' /proc/cpuinfo` - -If there is no output make sure that virtualization extensions is enabled in BIOS. -Verify that KVM modules are loaded in the kernel. -It should be loaded by default. Verify this using -`lsmod | grep kvm`. You should get some output. - -Running `modprobe kvm` might also help - -## Install dependencies - -`apt-get install software-properties-common` - -`add-apt-repository ppa:bitcoin/bitcoin` - -`apt-get update` - -``` -apt-get install git ruby sudo apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make curl build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils python3 libboost-all-dev libdb4.8-dev libdb4.8++-dev libminiupnpc-dev libzmq3-dev python-vm-builder apache2 apt-cacher-ng python-vm-builder qemu +```bash +sudo apt install git apt-cacher-ng ruby docker.io +sudo usermod -a -G docker $USER +exec su -l $USER #make effective the usermod command +cd ~/src +git clone https://github.com/project-ecc/eccoin.git +git clone https://github.com/devrandom/gitian-builder.git +cd gitian-builder +bin/make-base-vm --suite bionic --arch amd64 --docker ``` -git clone https://github.com/project-ecc/eccoin ~/eccoin - -checkout the branch of eccoin you plan on building - -git clone https://github.com/project-ecc/ecc-gitian-builder ~/ecc-gitian-builder - -mkdir ~/ecc-releases -mkdir ~/gitian.sigs +Build the binaries +------------------ -## Setup Gitian and build +These are the commands to actually roduce the executable: -#### Setup Gitian descriptors - -``` -export SIGNER=(your Gitian key, ie bluematt, sipa, etc) -export VERSION=(version) -export TAG=(tag or commit hash you want to build) -export ECC_PATH=~/eccoin -export RELEASE_DIR=~/ecc-releases/${VERSION} -``` - -#### Make the base gitian image - -Eccoin builds on an Ubuntu trusty VM, create that base vm image by doing the following: -``` -cd ecc-gitian-builder -./bin/make-base-vm --arch amd64 --suite trusty +```bash +cd ~/src/gitian-builder +export USE_DOCKER=1 +bin/gbuild -j 4 -m 10000 --url bitcoin=https://github.com/rpoject-ecc/eccoin.git --commit bitcoin=dev ../eccoin/contrib/gitian-descriptors/gitian-linux.yml ``` -If you get an error code here it is due to an issue with python. To fix it find the file - -`/usr/lib/python2.7/dist-packages/VMBuilder/plugins/ubuntu/dapper.py` - -Find the line: -``` -self.run_in_target('apt-get', '-y', '--force-yes', 'dist-upgrade', -``` -and replace it with: -``` -self.run_in_target('apt-get', '-y', '--force-yes', '--option=Dpkg::Options::=--force-confnew', 'dist-upgrade', -``` - -The full explaination of the fix can be found here: https://bugs.launchpad.net/vmbuilder/+bug/1659952 - -#### Perform the Build - -The formula for making a build is -``` -./bin/gbuild --url eccoin=${ECC_PATH} --commit eccoin=${TAG} ${ECC_PATH}/contrib/gitian-descriptors/(descriptor file) -``` - -For example if I wanted to build windows binaries: -``` -./bin/gbuild --url eccoin=${ECC_PATH} --commit eccoin=${TAG} ${ECC_PATH}/contrib/gitian-descriptors/gitian-win.yml -``` - -#### Signing - -Linux Signing - -``` -./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ~/gitian.sigs/ ${ECC_PATH}/contrib/gitian-descriptors/gitian-linux.yml -``` - -Windows Signing - -``` -./bin/gsign --signer $SIGNER --release ${VERSION}-win-unsigned --destination ~/gitian.sigs/ ${ECC_PATH}/contrib/gitian-descriptors/gitian-win.yml -``` - -MacOS Signing - -``` -./bin/gsign --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ~/gitian.sigs/ ${ECC_PATH}/contrib/gitian-descriptors/gitian-osx.yml -``` +Your binaries will be ready to be used in `build/out/` folder. diff --git a/src/Makefile.am b/src/Makefile.am index 7abf2cac..8581d35b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,4 +1,4 @@ -DIST_SUBDIRS = secp256k1 univalue rsm +DIST_SUBDIRS = secp256k1 univalue AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) AM_CXXFLAGS = $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index a67c775c..67ba78bb 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -95,7 +95,8 @@ BITCOIN_TESTS = \ BITCOIN_TESTS += \ rsm/test/rsm_promotion_tests.cpp \ rsm/test/rsm_simple_tests.cpp \ - rsm/test/rsm_starvation_tests.cpp + rsm/test/rsm_starvation_tests.cpp \ + rsm/test/test_cxx_rsm.h BITCOIN_TESTS += \ wallet/test/wallet_tests.cpp diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index 0228b821..f1846c10 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -1,23 +1,14 @@ -/* - * This file is part of the Eccoin project - * Copyright (c) 2009-2010 Satoshi Nakamoto - * Copyright (c) 2009-2016 The Bitcoin Core developers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ +// Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Unlimited developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif #include +#include #if defined(HAVE_SYS_SELECT_H) #include @@ -26,7 +17,6 @@ // Prior to GLIBC_2.14, memcpy was aliased to memmove. extern "C" void *memmove(void *a, const void *b, size_t c); extern "C" void *memcpy(void *a, const void *b, size_t c) { return memmove(a, b, c); } -#if defined(HAVE_SYS_SELECT_H) extern "C" void __chk_fail(void) __attribute__((__noreturn__)); extern "C" FDELT_TYPE __fdelt_warn(FDELT_TYPE a) { @@ -35,4 +25,52 @@ extern "C" FDELT_TYPE __fdelt_warn(FDELT_TYPE a) return a / __NFDBITS; } extern "C" FDELT_TYPE __fdelt_chk(FDELT_TYPE) __attribute__((weak, alias("__fdelt_warn"))); + +#if defined(__i386__) || defined(__arm__) + +extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t *rp); + +extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t *rp) +{ + int32_t c1 = 0, c2 = 0; + int64_t uu = u, vv = v; + int64_t w; + int64_t r; + + if (uu < 0) + { + c1 = ~c1, c2 = ~c2, uu = -uu; + } + if (vv < 0) + { + c1 = ~c1, vv = -vv; + } + + w = __udivmoddi4(uu, vv, (uint64_t *)&r); + if (c1) + w = -w; + if (c2) + r = -r; + + *rp = r; + return w; +} +#endif + +extern "C" float logf_old(float x); +extern "C" float log2f_old(float x); +#ifdef __i386__ +__asm(".symver logf_old,logf@GLIBC_2.0"); +__asm(".symver log2f_old,log2f@GLIBC_2.1"); +#elif defined(__amd64__) +__asm(".symver logf_old,logf@GLIBC_2.2.5"); +__asm(".symver log2f_old,log2f@GLIBC_2.2.5"); +#elif defined(__arm__) +__asm(".symver logf_old,logf@GLIBC_2.4"); +__asm(".symver log2f_old,log2f@GLIBC_2.4"); +#elif defined(__aarch64__) +__asm(".symver logf_old,logf@GLIBC_2.17"); +__asm(".symver log2f_old,log2f@GLIBC_2.17"); #endif +extern "C" float __wrap_logf(float x) { return logf_old(x); } +extern "C" float __wrap_log2f(float x) { return log2f_old(x); } diff --git a/src/rsm/test/rsm_promotion_tests.cpp b/src/rsm/test/rsm_promotion_tests.cpp index 9b7b61c5..4b1f9b5c 100644 --- a/src/rsm/test/rsm_promotion_tests.cpp +++ b/src/rsm/test/rsm_promotion_tests.cpp @@ -5,7 +5,7 @@ #include "recursive_shared_mutex.h" #include "test_cxx_rsm.h" -#include "timer.h" +#include "util/utiltime.h" #include diff --git a/src/rsm/test/rsm_simple_tests.cpp b/src/rsm/test/rsm_simple_tests.cpp index 4c7845f2..7a469ace 100644 --- a/src/rsm/test/rsm_simple_tests.cpp +++ b/src/rsm/test/rsm_simple_tests.cpp @@ -5,7 +5,7 @@ #include "recursive_shared_mutex.h" #include "test_cxx_rsm.h" -#include "timer.h" +#include "util/utiltime.h" #include diff --git a/src/rsm/test/rsm_starvation_tests.cpp b/src/rsm/test/rsm_starvation_tests.cpp index e9a9bc28..f77ea228 100644 --- a/src/rsm/test/rsm_starvation_tests.cpp +++ b/src/rsm/test/rsm_starvation_tests.cpp @@ -5,7 +5,7 @@ #include "recursive_shared_mutex.h" #include "test_cxx_rsm.h" -#include "timer.h" +#include "util/utiltime.h" #include From a5f75659c480ffeb48c7269ee8a59097d5889e8a Mon Sep 17 00:00:00 2001 From: Griffith Date: Wed, 1 May 2019 02:25:53 +0900 Subject: [PATCH 40/46] adjust sync_blocks calls in mintingtest.py (#164) --- qa/rpc-tests/mintingtest.py | 5 +++-- src/wallet/wallet.cpp | 9 --------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/qa/rpc-tests/mintingtest.py b/qa/rpc-tests/mintingtest.py index f731bfb6..3d574ec9 100755 --- a/qa/rpc-tests/mintingtest.py +++ b/qa/rpc-tests/mintingtest.py @@ -50,10 +50,11 @@ def run_test(self): # generate non-empty blocks on the mining node for x in range(0, self.generatedblocks): self.nodes[0].generate(1) + if x % 25 == 0: + self.sync_blocks() self.sync_blocks() - - time.sleep(45) + time.sleep(25) # check that nodes across different db's have the same data for x in range(0, self.blockstotest): diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 4f0091ef..f979bdd8 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3470,15 +3470,6 @@ bool CWallet::CreateCoinStake(const CKeyStore &keystore, fKernelFound = true; break; } - else - { - /// TODO - // sleep very briefly before trying next pcoin for kernel. removing this would be an obvious opimization - // but it is here to keep cpu temps - // lower in times of high difficulty staking, possible change to configurable parameter later so i - // will mark this with todo - MilliSleep(5); - } } if (fKernelFound) break; // if kernel is found stop searching From e0d343a081f3acaa6d098b0e2e2205ca6ce6aaf1 Mon Sep 17 00:00:00 2001 From: Griffith Date: Wed, 1 May 2019 06:01:09 +0900 Subject: [PATCH 41/46] bump version to 0.2.5.15 with release false for rc1 (#165) [skip ci] --- configure.ac | 6 +++--- contrib/gitian-descriptors/gitian-arm.yml | 2 +- contrib/gitian-descriptors/gitian-linux.yml | 2 +- contrib/gitian-descriptors/gitian-osx.yml | 2 +- contrib/gitian-descriptors/gitian-win.yml | 2 +- src/clientversion.h | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/configure.ac b/configure.ac index e8fc55a8..574dc1e9 100644 --- a/configure.ac +++ b/configure.ac @@ -12,9 +12,9 @@ AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 0) define(_CLIENT_VERSION_MINOR, 2) define(_CLIENT_VERSION_REVISION, 5) -define(_CLIENT_VERSION_BUILD, 14) # version 99 here indicates an unreleased version -define(_CLIENT_VERSION_IS_RELEASE, true) -define(_COPYRIGHT_YEAR, 2018) +define(_CLIENT_VERSION_BUILD, 15) # version 99 here indicates an unreleased version +define(_CLIENT_VERSION_IS_RELEASE, false) +define(_COPYRIGHT_YEAR, 2019) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[Eccoin Developers]) AC_INIT([ECCOIN],[_CLIENT_VERSION_MAJOR._CLIENT_VERSION_MINOR._CLIENT_VERSION_REVISION._CLIENT_VERSION_BUILD],[https://github.com/project-ecc/eccoin/issues],[eccoin]) diff --git a/contrib/gitian-descriptors/gitian-arm.yml b/contrib/gitian-descriptors/gitian-arm.yml index e4f4ec67..e437d7ba 100644 --- a/contrib/gitian-descriptors/gitian-arm.yml +++ b/contrib/gitian-descriptors/gitian-arm.yml @@ -1,5 +1,5 @@ --- -name: "eccoin-linux-0.2.5.12" +name: "eccoin-linux-0.2.5.15" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 1437195d..a51acb4d 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "eccoin-linux-0.2.5.12" +name: "eccoin-linux-0.2.5.15" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 41ff94c3..c261ab58 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,5 +1,5 @@ --- -name: "eccoin-osx-0.2.5.12" +name: "eccoin-osx-0.2.5.15" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 3661764d..35e02511 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,5 +1,5 @@ --- -name: "eccoin-win-0.2.5.12" +name: "eccoin-win-0.2.5.15" enable_cache: true suites: - "bionic" diff --git a/src/clientversion.h b/src/clientversion.h index 7a7dde63..8ba8426f 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -34,16 +34,16 @@ #define CLIENT_VERSION_MAJOR 0 #define CLIENT_VERSION_MINOR 2 #define CLIENT_VERSION_REVISION 5 -#define CLIENT_VERSION_BUILD 14 +#define CLIENT_VERSION_BUILD 15 //! Set to true for release, false for prerelease or test build -#define CLIENT_VERSION_IS_RELEASE true +#define CLIENT_VERSION_IS_RELEASE false /** * Copyright year (2009-this) * Todo: update this when changing our copyright comments in the source */ -#define COPYRIGHT_YEAR 2018 +#define COPYRIGHT_YEAR 2019 #endif // HAVE_CONFIG_H From 1becbcfeb3ae93b0746639f88fda83b5a908c5a6 Mon Sep 17 00:00:00 2001 From: Griffith Date: Wed, 1 May 2019 21:44:47 +0900 Subject: [PATCH 42/46] add proper destructor to thread_group, disable default constructor (#166) --- src/net/net.cpp | 1 - src/threadgroup.h | 32 ++++++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/net/net.cpp b/src/net/net.cpp index 2abe4eb3..6db5ab4f 100644 --- a/src/net/net.cpp +++ b/src/net/net.cpp @@ -2436,7 +2436,6 @@ extern int initMaxConnections; bool CConnman::Start(std::string &strNodeError) { - netThreads.clear(); nTotalBytesRecv = 0; nTotalBytesSent = 0; nMaxOutboundTotalBytesSentInCycle = 0; diff --git a/src/threadgroup.h b/src/threadgroup.h index 38a4c422..2d5e8c5c 100644 --- a/src/threadgroup.h +++ b/src/threadgroup.h @@ -12,39 +12,43 @@ class thread_group { private: std::vector threads; - std::vector names; std::atomic *killswitch; + // disable default constructor + thread_group(){} public: - thread_group(std::atomic *_killswitch) { killswitch = _killswitch; } + thread_group(std::atomic *_killswitch) + { + threads.clear(); + killswitch = _killswitch; + } void interrupt_all() { killswitch->store(true); } template void create_thread(std::string name, Fn &&f, Args &&... args) { threads.push_back(std::thread(f, args...)); - names.push_back(name); - } - - void clear() - { - interrupt_all(); - join_all(); - threads.clear(); - names.clear(); } bool empty() { return threads.empty(); } void join_all() { - // printf("num threads = %u \n", threads.size()); for (size_t i = 0; i < threads.size(); i++) { - // printf("trying to join thread %s \n", names[i].c_str()); if (threads[i].joinable()) { threads[i].join(); } - // printf("joined thread %s \n", names[i].c_str()); + } + threads.clear(); + } + + ~thread_group() + { + interrupt_all(); + for (auto &t : threads) + { + if (t.joinable()) + t.detach(); } } }; From 864baf3dd474da2c2020f61f790ae4f69f46eecb Mon Sep 17 00:00:00 2001 From: Griffith Date: Thu, 2 May 2019 04:47:42 +0900 Subject: [PATCH 43/46] Close DB on error (#167) --- src/wallet/db.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 5b42623f..91069ed9 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -111,7 +111,10 @@ bool CDBEnv::Open(const fs::path &pathIn) DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags, S_IRUSR | S_IWUSR); if (ret != 0) + { + dbenv->close(0); return error("CDBEnv::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret)); + } fDbEnvInit = true; fMockDb = false; @@ -377,11 +380,13 @@ bool CDB::Rewrite(const std::string &strFile, const char *pszSkip) if (ret > 0) { LogPrintf("CDB::Rewrite: Can't create database file %s\n", strFileRes); + pdbCopy->close(0); fSuccess = false; } Dbc *pcursor = db.GetCursor(); if (pcursor) + { while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); @@ -412,12 +417,19 @@ bool CDB::Rewrite(const std::string &strFile, const char *pszSkip) if (ret2 > 0) fSuccess = false; } + } if (fSuccess) { db.Close(); bitdb.CloseDb(strFile); if (pdbCopy->close(0)) + { fSuccess = false; + } + else + { + pdbCopy->close(0); + } delete pdbCopy; } } From 9468f96b32cb847dc852735383f2ef9c688c5ee5 Mon Sep 17 00:00:00 2001 From: Griffith Date: Fri, 3 May 2019 09:14:55 +0900 Subject: [PATCH 44/46] Osx gitian fix (#168) * Add ca-certificates to osx gitian descriptor * remove names from threads due to osx compilation complaints --- contrib/gitian-descriptors/gitian-osx.yml | 1 + src/blockgeneration/blockgeneration.cpp | 4 ++-- src/init.cpp | 7 +++---- src/net/net.cpp | 12 ++++++------ src/threadgroup.h | 7 +++---- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index c261ab58..bde59fb4 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -26,6 +26,7 @@ packages: - "python-dev" - "python-setuptools" - "fonts-tuffy" +- "ca-certificates" reference_datetime: "2018-10-12 00:00:00" remotes: - "url": "https://github.com/project-ecc/eccoin.git" diff --git a/src/blockgeneration/blockgeneration.cpp b/src/blockgeneration/blockgeneration.cpp index 8e7664e3..5a350494 100644 --- a/src/blockgeneration/blockgeneration.cpp +++ b/src/blockgeneration/blockgeneration.cpp @@ -86,7 +86,7 @@ void ThreadMiner(void *parg, bool shutdownOnly) CWallet *pwallet = (CWallet *)parg; try { - minerThreads->create_thread("eccMiner", &EccMiner, pwallet); + minerThreads->create_thread(&EccMiner, pwallet); } catch (std::exception &e) { @@ -120,7 +120,7 @@ void ThreadMinter(void *parg, bool shutdownOnly) CWallet *pwallet = (CWallet *)parg; try { - minterThreads->create_thread("eccMinter", &EccMinter, pwallet); + minterThreads->create_thread(&EccMinter, pwallet); } catch (std::exception &e) { diff --git a/src/init.cpp b/src/init.cpp index 76b1d157..4c948221 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1192,8 +1192,7 @@ bool AppInit2(thread_group &threadGroup) { for (int i = 0; i < nScriptCheckThreads - 1; i++) { - std::string name = "scriptCheck" + std::to_string(i); - threadGroup.create_thread(name, &ThreadScriptCheck); + threadGroup.create_thread(&ThreadScriptCheck); } } @@ -1606,7 +1605,7 @@ bool AppInit2(thread_group &threadGroup) for (auto const &strFile : gArgs.GetArgs("-loadblock")) vImportFiles.push_back(strFile); } - threadGroup.create_thread("importFiles", &ThreadImport, vImportFiles); + threadGroup.create_thread(&ThreadImport, vImportFiles); if (pnetMan->getChainActive()->chainActive.Tip() == nullptr) { @@ -1679,7 +1678,7 @@ bool AppInit2(thread_group &threadGroup) pwalletMain->ReacceptWalletTransactions(); // Run a thread to flush wallet periodically - threadGroup.create_thread("flushWalletDB", &ThreadFlushWalletDB, boost::ref(pwalletMain->strWalletFile)); + threadGroup.create_thread(&ThreadFlushWalletDB, boost::ref(pwalletMain->strWalletFile)); } return !shutdown_threads.load(); diff --git a/src/net/net.cpp b/src/net/net.cpp index 6db5ab4f..eee5b10b 100644 --- a/src/net/net.cpp +++ b/src/net/net.cpp @@ -2531,7 +2531,7 @@ bool CConnman::Start(std::string &strNodeError) interruptNet.store(false); // Send and receive from sockets, accept connections - netThreads.create_thread("net", &CConnman::ThreadSocketHandler, this); + netThreads.create_thread(&CConnman::ThreadSocketHandler, this); if (!gArgs.GetBoolArg("-dnsseed", true)) { @@ -2539,23 +2539,23 @@ bool CConnman::Start(std::string &strNodeError) } else { - netThreads.create_thread("dnsseed", &CConnman::ThreadDNSAddressSeed, this); + netThreads.create_thread(&CConnman::ThreadDNSAddressSeed, this); } // Initiate outbound connections from -addnode - netThreads.create_thread("addcon", &CConnman::ThreadOpenAddedConnections, this); + netThreads.create_thread(&CConnman::ThreadOpenAddedConnections, this); // Initiate outbound connections unless connect=0 if (!gArgs.IsArgSet("-connect") || gArgs.GetArgs("-connect").size() != 1 || gArgs.GetArgs("-connect")[0] != "0") { - netThreads.create_thread("opencon", &CConnman::ThreadOpenConnections, this); + netThreads.create_thread(&CConnman::ThreadOpenConnections, this); } // Process messages - netThreads.create_thread("msghand", &CConnman::ThreadMessageHandler, this); + netThreads.create_thread(&CConnman::ThreadMessageHandler, this); // Dump network addresses - netThreads.create_thread("dumpdata", &CConnman::DumpData, this, DUMP_ADDRESSES_INTERVAL); + netThreads.create_thread(&CConnman::DumpData, this, DUMP_ADDRESSES_INTERVAL); return true; } diff --git a/src/threadgroup.h b/src/threadgroup.h index 2d5e8c5c..bf597db9 100644 --- a/src/threadgroup.h +++ b/src/threadgroup.h @@ -14,8 +14,7 @@ class thread_group std::vector threads; std::atomic *killswitch; // disable default constructor - thread_group(){} - + thread_group() {} public: thread_group(std::atomic *_killswitch) { @@ -24,7 +23,7 @@ class thread_group } void interrupt_all() { killswitch->store(true); } template - void create_thread(std::string name, Fn &&f, Args &&... args) + void create_thread(Fn &&f, Args &&... args) { threads.push_back(std::thread(f, args...)); } @@ -41,7 +40,7 @@ class thread_group } threads.clear(); } - + ~thread_group() { interrupt_all(); From 58f343b82a9eb0e4f90cc858f368b163ca3301c5 Mon Sep 17 00:00:00 2001 From: Griffith Date: Fri, 3 May 2019 09:59:15 +0900 Subject: [PATCH 45/46] pull down leveldb changes from upstream (#169) --- src/leveldb/db/db_impl.cc | 2 +- src/leveldb/db/dbformat.h | 4 ++-- src/leveldb/db/leveldbutil.cc | 1 + src/leveldb/db/log_reader.cc | 2 +- src/leveldb/db/repair.cc | 2 +- src/leveldb/helpers/memenv/memenv.cc | 3 +++ src/leveldb/include/leveldb/env.h | 9 +++++++++ src/leveldb/table/format.cc | 10 +++++----- src/leveldb/util/env_posix.cc | 12 ++++++++++-- src/leveldb/util/env_win.cc | 3 +++ src/leveldb/util/logging.cc | 2 +- 11 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc index f43ad767..3bb58e56 100644 --- a/src/leveldb/db/db_impl.cc +++ b/src/leveldb/db/db_impl.cc @@ -414,7 +414,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, status.ok()) { if (record.size() < 12) { reporter.Corruption( - record.size(), Status::Corruption("log record too small")); + record.size(), Status::Corruption("log record too small", fname)); continue; } WriteBatchInternal::SetContents(&batch, record); diff --git a/src/leveldb/db/dbformat.h b/src/leveldb/db/dbformat.h index ea897b13..a05ded14 100644 --- a/src/leveldb/db/dbformat.h +++ b/src/leveldb/db/dbformat.h @@ -25,10 +25,10 @@ static const int kNumLevels = 7; static const int kL0_CompactionTrigger = 4; // Soft limit on number of level-0 files. We slow down writes at this point. -static const int kL0_SlowdownWritesTrigger = 8; +static const int kL0_SlowdownWritesTrigger = 4092; // Maximum number of level-0 files. We stop writes at this point. -static const int kL0_StopWritesTrigger = 12; +static const int kL0_StopWritesTrigger = 4096; // Maximum level to which a new compacted memtable is pushed if it // does not create overlap. We try to push to level 2 to avoid the diff --git a/src/leveldb/db/leveldbutil.cc b/src/leveldb/db/leveldbutil.cc index 9f4b7dd7..d06d64d6 100644 --- a/src/leveldb/db/leveldbutil.cc +++ b/src/leveldb/db/leveldbutil.cc @@ -19,6 +19,7 @@ class StdoutPrinter : public WritableFile { virtual Status Close() { return Status::OK(); } virtual Status Flush() { return Status::OK(); } virtual Status Sync() { return Status::OK(); } + virtual std::string GetName() const { return "[stdout]"; } }; bool HandleDumpCommand(Env* env, char** files, int num) { diff --git a/src/leveldb/db/log_reader.cc b/src/leveldb/db/log_reader.cc index a6d30454..8b6ad136 100644 --- a/src/leveldb/db/log_reader.cc +++ b/src/leveldb/db/log_reader.cc @@ -186,7 +186,7 @@ uint64_t Reader::LastRecordOffset() { } void Reader::ReportCorruption(uint64_t bytes, const char* reason) { - ReportDrop(bytes, Status::Corruption(reason)); + ReportDrop(bytes, Status::Corruption(reason, file_->GetName())); } void Reader::ReportDrop(uint64_t bytes, const Status& reason) { diff --git a/src/leveldb/db/repair.cc b/src/leveldb/db/repair.cc index 4cd4bb04..7281e3d3 100644 --- a/src/leveldb/db/repair.cc +++ b/src/leveldb/db/repair.cc @@ -203,7 +203,7 @@ class Repairer { while (reader.ReadRecord(&record, &scratch)) { if (record.size() < 12) { reporter.Corruption( - record.size(), Status::Corruption("log record too small")); + record.size(), Status::Corruption("log record too small", logname)); continue; } WriteBatchInternal::SetContents(&batch, record); diff --git a/src/leveldb/helpers/memenv/memenv.cc b/src/leveldb/helpers/memenv/memenv.cc index 9a98884d..68c0614a 100644 --- a/src/leveldb/helpers/memenv/memenv.cc +++ b/src/leveldb/helpers/memenv/memenv.cc @@ -176,6 +176,7 @@ class SequentialFileImpl : public SequentialFile { return Status::OK(); } + virtual std::string GetName() const { return "[memenv]"; } private: FileState* file_; uint64_t pos_; @@ -196,6 +197,7 @@ class RandomAccessFileImpl : public RandomAccessFile { return file_->Read(offset, n, result, scratch); } + virtual std::string GetName() const { return "[memenv]"; } private: FileState* file_; }; @@ -218,6 +220,7 @@ class WritableFileImpl : public WritableFile { virtual Status Flush() { return Status::OK(); } virtual Status Sync() { return Status::OK(); } + virtual std::string GetName() const { return "[memenv]"; } private: FileState* file_; }; diff --git a/src/leveldb/include/leveldb/env.h b/src/leveldb/include/leveldb/env.h index 99b6c214..275d441e 100644 --- a/src/leveldb/include/leveldb/env.h +++ b/src/leveldb/include/leveldb/env.h @@ -191,6 +191,9 @@ class SequentialFile { // REQUIRES: External synchronization virtual Status Skip(uint64_t n) = 0; + // Get a name for the file, only for error reporting + virtual std::string GetName() const = 0; + private: // No copying allowed SequentialFile(const SequentialFile&); @@ -215,6 +218,9 @@ class RandomAccessFile { virtual Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const = 0; + // Get a name for the file, only for error reporting + virtual std::string GetName() const = 0; + private: // No copying allowed RandomAccessFile(const RandomAccessFile&); @@ -234,6 +240,9 @@ class WritableFile { virtual Status Flush() = 0; virtual Status Sync() = 0; + // Get a name for the file, only for error reporting + virtual std::string GetName() const = 0; + private: // No copying allowed WritableFile(const WritableFile&); diff --git a/src/leveldb/table/format.cc b/src/leveldb/table/format.cc index 24e4e024..285e1c0d 100644 --- a/src/leveldb/table/format.cc +++ b/src/leveldb/table/format.cc @@ -82,7 +82,7 @@ Status ReadBlock(RandomAccessFile* file, } if (contents.size() != n + kBlockTrailerSize) { delete[] buf; - return Status::Corruption("truncated block read"); + return Status::Corruption("truncated block read", file->GetName()); } // Check the crc of the type and the block contents @@ -92,7 +92,7 @@ Status ReadBlock(RandomAccessFile* file, const uint32_t actual = crc32c::Value(data, n + 1); if (actual != crc) { delete[] buf; - s = Status::Corruption("block checksum mismatch"); + s = Status::Corruption("block checksum mismatch", file->GetName()); return s; } } @@ -119,13 +119,13 @@ Status ReadBlock(RandomAccessFile* file, size_t ulength = 0; if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) { delete[] buf; - return Status::Corruption("corrupted compressed block contents"); + return Status::Corruption("corrupted compressed block contents", file->GetName()); } char* ubuf = new char[ulength]; if (!port::Snappy_Uncompress(data, n, ubuf)) { delete[] buf; delete[] ubuf; - return Status::Corruption("corrupted compressed block contents"); + return Status::Corruption("corrupted compressed block contents", file->GetName()); } delete[] buf; result->data = Slice(ubuf, ulength); @@ -135,7 +135,7 @@ Status ReadBlock(RandomAccessFile* file, } default: delete[] buf; - return Status::Corruption("bad block type"); + return Status::Corruption("bad block type", file->GetName()); } return Status::OK(); diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index dd852af3..f7791831 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -121,6 +121,8 @@ class PosixSequentialFile: public SequentialFile { } return Status::OK(); } + + virtual std::string GetName() const { return filename_; } }; // pread() based random-access @@ -172,6 +174,8 @@ class PosixRandomAccessFile: public RandomAccessFile { } return s; } + + virtual std::string GetName() const { return filename_; } }; // mmap() based random-access @@ -206,6 +210,8 @@ class PosixMmapReadableFile: public RandomAccessFile { } return s; } + + virtual std::string GetName() const { return filename_; } }; class PosixWritableFile : public WritableFile { @@ -287,6 +293,8 @@ class PosixWritableFile : public WritableFile { } return s; } + + virtual std::string GetName() const { return filename_; } }; static int LockOrUnlock(int fd, bool lock) { @@ -577,8 +585,8 @@ static int MaxMmaps() { if (mmap_limit >= 0) { return mmap_limit; } - // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes. - mmap_limit = sizeof(void*) >= 8 ? 1000 : 0; + // Up to 4096 mmaps for 64-bit binaries; none for smaller pointer sizes. + mmap_limit = sizeof(void*) >= 8 ? 4096 : 0; return mmap_limit; } diff --git a/src/leveldb/util/env_win.cc b/src/leveldb/util/env_win.cc index d32c4e67..81380216 100644 --- a/src/leveldb/util/env_win.cc +++ b/src/leveldb/util/env_win.cc @@ -78,6 +78,7 @@ class Win32SequentialFile : public SequentialFile virtual Status Read(size_t n, Slice* result, char* scratch); virtual Status Skip(uint64_t n); BOOL isEnable(); + virtual std::string GetName() const { return _filename; } private: BOOL _Init(); void _CleanUp(); @@ -94,6 +95,7 @@ class Win32RandomAccessFile : public RandomAccessFile virtual ~Win32RandomAccessFile(); virtual Status Read(uint64_t offset, size_t n, Slice* result,char* scratch) const; BOOL isEnable(); + virtual std::string GetName() const { return _filename; } private: BOOL _Init(LPCWSTR path); void _CleanUp(); @@ -114,6 +116,7 @@ class Win32WritableFile : public WritableFile virtual Status Flush(); virtual Status Sync(); BOOL isEnable(); + virtual std::string GetName() const { return filename_; } private: std::string filename_; ::HANDLE _hFile; diff --git a/src/leveldb/util/logging.cc b/src/leveldb/util/logging.cc index db6160c8..6995d902 100644 --- a/src/leveldb/util/logging.cc +++ b/src/leveldb/util/logging.cc @@ -52,7 +52,7 @@ bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { unsigned char c = (*in)[0]; if (c >= '0' && c <= '9') { ++digits; - const int delta = (c - '0'); + const unsigned int delta = (c - '0'); static const uint64_t kMaxUint64 = ~static_cast(0); if (v > kMaxUint64/10 || (v == kMaxUint64/10 && delta > kMaxUint64%10)) { From d201d0119185ee7c5d388dc35350337ac9ea2f2f Mon Sep 17 00:00:00 2001 From: Griffith Date: Fri, 3 May 2019 18:31:36 +0900 Subject: [PATCH 46/46] set release for version 0.2.5.15 (#170) --- configure.ac | 2 +- src/clientversion.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 574dc1e9..422c2504 100644 --- a/configure.ac +++ b/configure.ac @@ -13,7 +13,7 @@ define(_CLIENT_VERSION_MAJOR, 0) define(_CLIENT_VERSION_MINOR, 2) define(_CLIENT_VERSION_REVISION, 5) define(_CLIENT_VERSION_BUILD, 15) # version 99 here indicates an unreleased version -define(_CLIENT_VERSION_IS_RELEASE, false) +define(_CLIENT_VERSION_IS_RELEASE, true) define(_COPYRIGHT_YEAR, 2019) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[Eccoin Developers]) diff --git a/src/clientversion.h b/src/clientversion.h index 8ba8426f..999cc280 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -37,7 +37,7 @@ #define CLIENT_VERSION_BUILD 15 //! Set to true for release, false for prerelease or test build -#define CLIENT_VERSION_IS_RELEASE false +#define CLIENT_VERSION_IS_RELEASE true /** * Copyright year (2009-this)