diff options
Diffstat (limited to 'src')
50 files changed, 1917 insertions, 662 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index e7f1d82b8b..5a5e3abcfa 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -87,7 +87,6 @@ BITCOIN_CORE_H = \ checkpoints.h \ checkqueue.h \ clientversion.h \ - coincontrol.h \ coins.h \ compat.h \ compat/byteswap.h \ @@ -133,7 +132,7 @@ BITCOIN_CORE_H = \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ support/cleanse.h \ - support/pagelocker.h \ + support/lockedpool.h \ sync.h \ threadsafety.h \ timedata.h \ @@ -147,6 +146,7 @@ BITCOIN_CORE_H = \ utiltime.h \ validationinterface.h \ versionbits.h \ + wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ wallet/rpcwallet.h \ @@ -310,7 +310,7 @@ libbitcoin_common_a_SOURCES = \ libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ - support/pagelocker.cpp \ + support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ compat/glibc_sanity.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c83432e91a..9760ad089c 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -17,7 +17,8 @@ bench_bench_bitcoin_SOURCES = \ bench/ccoins_caching.cpp \ bench/mempool_eviction.cpp \ bench/verify_script.cpp \ - bench/base58.cpp + bench/base58.cpp \ + bench/lockedpool.cpp bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp new file mode 100644 index 0000000000..5df5b1ac6e --- /dev/null +++ b/src/bench/lockedpool.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" + +#include "support/lockedpool.h" + +#include <iostream> +#include <vector> + +#define ASIZE 2048 +#define BITER 5000 +#define MSIZE 2048 + +static void LockedPool(benchmark::State& state) +{ + void *synth_base = reinterpret_cast<void*>(0x08000000); + const size_t synth_size = 1024*1024; + Arena b(synth_base, synth_size, 16); + + std::vector<void*> addr; + for (int x=0; x<ASIZE; ++x) + addr.push_back(0); + uint32_t s = 0x12345678; + while (state.KeepRunning()) { + for (int x=0; x<BITER; ++x) { + int idx = s & (addr.size()-1); + if (s & 0x80000000) { + b.free(addr[idx]); + addr[idx] = 0; + } else if(!addr[idx]) { + addr[idx] = b.alloc((s >> 16) & (MSIZE-1)); + } + bool lsb = s & 1; + s >>= 1; + if (lsb) + s ^= 0xf00f00f0; // LFSR period 0xf7ffffe0 + } + } + for (void *ptr: addr) + b.free(ptr); + addr.clear(); +} + +BENCHMARK(LockedPool); + diff --git a/src/bloom.cpp b/src/bloom.cpp index 2677652ada..d00befc61c 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -34,7 +34,7 @@ CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int * See https://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas */ isFull(false), - isEmpty(false), + isEmpty(true), nHashFuncs(min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)), nTweak(nTweakIn), nFlags(nFlagsIn) diff --git a/src/chain.cpp b/src/chain.cpp index 77e924e703..1e611906d1 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -61,6 +61,13 @@ const CBlockIndex *CChain::FindFork(const CBlockIndex *pindex) const { return pindex; } +CBlockIndex* CChain::FindLatestBefore(int64_t nTime) const +{ + std::vector<CBlockIndex*>::const_iterator lower = std::lower_bound(vChain.begin(), vChain.end(), nTime, + [](CBlockIndex* pBlock, const int64_t& time) -> bool { return pBlock->GetBlockTime() < time; }); + return (lower == vChain.end() ? NULL : *lower); +} + /** Turn the lowest '1' bit in the binary representation of a number into a '0'. */ int static inline InvertLowestOne(int n) { return n & (n - 1); } diff --git a/src/chain.h b/src/chain.h index e2f8c56522..46a16a3061 100644 --- a/src/chain.h +++ b/src/chain.h @@ -459,6 +459,9 @@ public: /** Find the last common block between this chain and a block index entry. */ const CBlockIndex *FindFork(const CBlockIndex *pindex) const; + + /** Find the most recent block with timestamp lower than the given. */ + CBlockIndex* FindLatestBefore(int64_t nTime) const; }; #endif // BITCOIN_CHAIN_H diff --git a/src/init.cpp b/src/init.cpp index 9665d0e06d..008bb6fae5 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -207,6 +207,7 @@ void Shutdown() StopTorControl(); UnregisterNodeSignals(GetNodeSignals()); + DumpMempool(); if (fFeeEstimatesInitialized) { @@ -359,13 +360,13 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-banscore=<n>", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt("-bantime=<n>", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=<addr>", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s)")); + strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -noconnect or -connect=0 alone to disable automatic connections")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); - strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect)")); + strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect/-noconnect)")); strUsage += HelpMessageOpt("-externalip=<ip>", _("Specify your own public address")); strUsage += HelpMessageOpt("-forcednsseed", strprintf(_("Always query for peer addresses via DNS lookup (default: %u)"), DEFAULT_FORCEDNSSEED)); - strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect)")); + strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect/-noconnect)")); strUsage += HelpMessageOpt("-listenonion", strprintf(_("Automatically create Tor hidden service (default: %d)"), DEFAULT_LISTEN_ONION)); strUsage += HelpMessageOpt("-maxconnections=<n>", strprintf(_("Maintain at most <n> connections to peers (default: %u)"), DEFAULT_MAX_PEER_CONNECTIONS)); strUsage += HelpMessageOpt("-maxreceivebuffer=<n>", strprintf(_("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)"), DEFAULT_MAXRECEIVEBUFFER)); @@ -659,6 +660,8 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) LogPrintf("Stopping after block import\n"); StartShutdown(); } + + LoadMempool(); } /** Sanity checks @@ -960,8 +963,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) ::minRelayTxFee = CFeeRate(n); } - fRequireStandard = !GetBoolArg("-acceptnonstdtxn", !Params().RequireStandard()); - if (Params().RequireStandard() && !fRequireStandard) + fRequireStandard = !GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); + if (chainparams.RequireStandard() && !fRequireStandard) return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); nBytesPerSigOp = GetArg("-bytespersigop", nBytesPerSigOp); @@ -996,7 +999,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (!mapMultiArgs["-bip9params"].empty()) { // Allow overriding BIP9 parameters for testing - if (!Params().MineBlocksOnDemand()) { + if (!chainparams.MineBlocksOnDemand()) { return InitError("BIP9 parameters may only be overridden on regtest."); } const vector<string>& deployments = mapMultiArgs["-bip9params"]; @@ -1320,7 +1323,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) CleanupBlockRevFiles(); } - if (!LoadBlockIndex()) { + if (!LoadBlockIndex(chainparams)) { strLoadError = _("Error loading block database"); break; } @@ -1447,7 +1450,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (Params().GetConsensus().vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout != 0) { + if (chainparams.GetConsensus().vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout != 0) { // Only advertize witness capabilities if they have a reasonable start time. // This allows us to have the code merged without a defined softfork, by setting its // end time to 0. diff --git a/src/key.cpp b/src/key.cpp index aae9b042ac..b3ea98fb92 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -125,8 +125,8 @@ bool CKey::Check(const unsigned char *vch) { void CKey::MakeNewKey(bool fCompressedIn) { do { - GetStrongRandBytes(vch, sizeof(vch)); - } while (!Check(vch)); + GetStrongRandBytes(keydata.data(), keydata.size()); + } while (!Check(keydata.data())); fValid = true; fCompressed = fCompressedIn; } @@ -224,20 +224,18 @@ bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck=false) { bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert(IsCompressed()); - unsigned char out[64]; - LockObject(out); + std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); if ((nChild >> 31) == 0) { CPubKey pubkey = GetPubKey(); assert(pubkey.begin() + 33 == pubkey.end()); - BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, out); + BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, vout.data()); } else { assert(begin() + 32 == end()); - BIP32Hash(cc, nChild, 0, begin(), out); + BIP32Hash(cc, nChild, 0, begin(), vout.data()); } - memcpy(ccChild.begin(), out+32, 32); + memcpy(ccChild.begin(), vout.data()+32, 32); memcpy((unsigned char*)keyChild.begin(), begin(), 32); - bool ret = secp256k1_ec_privkey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), out); - UnlockObject(out); + bool ret = secp256k1_ec_privkey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), vout.data()); keyChild.fCompressed = true; keyChild.fValid = ret; return ret; @@ -253,12 +251,10 @@ bool CExtKey::Derive(CExtKey &out, unsigned int _nChild) const { void CExtKey::SetMaster(const unsigned char *seed, unsigned int nSeedLen) { static const unsigned char hashkey[] = {'B','i','t','c','o','i','n',' ','s','e','e','d'}; - unsigned char out[64]; - LockObject(out); - CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(out); - key.Set(&out[0], &out[32], true); - memcpy(chaincode.begin(), &out[32], 32); - UnlockObject(out); + std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); + CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(vout.data()); + key.Set(&vout[0], &vout[32], true); + memcpy(chaincode.begin(), &vout[32], 32); nDepth = 0; nChild = 0; memset(vchFingerprint, 0, sizeof(vchFingerprint)); @@ -308,12 +304,10 @@ void ECC_Start() { { // Pass in a random blinding seed to the secp256k1 context. - unsigned char seed[32]; - LockObject(seed); - GetRandBytes(seed, 32); - bool ret = secp256k1_context_randomize(ctx, seed); + std::vector<unsigned char, secure_allocator<unsigned char>> vseed(32); + GetRandBytes(vseed.data(), 32); + bool ret = secp256k1_context_randomize(ctx, vseed.data()); assert(ret); - UnlockObject(seed); } secp256k1_context_sign = ctx; @@ -43,9 +43,7 @@ private: bool fCompressed; //! The actual byte data - unsigned char vch[32]; - - static_assert(sizeof(vch) == 32, "vch must be 32 bytes in length to not break serialization"); + std::vector<unsigned char, secure_allocator<unsigned char> > keydata; //! Check whether the 32-byte array pointed to be vch is valid keydata. bool static Check(const unsigned char* vch); @@ -54,37 +52,30 @@ public: //! Construct an invalid private key. CKey() : fValid(false), fCompressed(false) { - LockObject(vch); - } - - //! Copy constructor. This is necessary because of memlocking. - CKey(const CKey& secret) : fValid(secret.fValid), fCompressed(secret.fCompressed) - { - LockObject(vch); - memcpy(vch, secret.vch, sizeof(vch)); + // Important: vch must be 32 bytes in length to not break serialization + keydata.resize(32); } //! Destructor (again necessary because of memlocking). ~CKey() { - UnlockObject(vch); } friend bool operator==(const CKey& a, const CKey& b) { return a.fCompressed == b.fCompressed && a.size() == b.size() && - memcmp(&a.vch[0], &b.vch[0], a.size()) == 0; + memcmp(a.keydata.data(), b.keydata.data(), a.size()) == 0; } //! Initialize using begin and end iterators to byte data. template <typename T> void Set(const T pbegin, const T pend, bool fCompressedIn) { - if (pend - pbegin != sizeof(vch)) { + if (size_t(pend - pbegin) != keydata.size()) { fValid = false; } else if (Check(&pbegin[0])) { - memcpy(vch, (unsigned char*)&pbegin[0], sizeof(vch)); + memcpy(keydata.data(), (unsigned char*)&pbegin[0], keydata.size()); fValid = true; fCompressed = fCompressedIn; } else { @@ -93,9 +84,9 @@ public: } //! Simple read-only vector-like interface. - unsigned int size() const { return (fValid ? sizeof(vch) : 0); } - const unsigned char* begin() const { return vch; } - const unsigned char* end() const { return vch + size(); } + unsigned int size() const { return (fValid ? keydata.size() : 0); } + const unsigned char* begin() const { return keydata.data(); } + const unsigned char* end() const { return keydata.data() + size(); } //! Check whether this private key is valid. bool IsValid() const { return fValid; } diff --git a/src/main.cpp b/src/main.cpp index 50158b4687..4cc886afb5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1135,7 +1135,7 @@ std::string FormatStateMessage(const CValidationState &state) } bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransaction& tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) { const uint256 hash = tx.GetHash(); @@ -1308,7 +1308,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); + CTxMemPoolEntry entry(tx, nFees, nAcceptTime, dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); unsigned int nSize = entry.GetTxSize(); // Check that the transaction doesn't have an excessive number of @@ -1572,11 +1572,11 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C return true; } -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); if (!res) { BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) pcoinsTip->Uncache(hashTx); @@ -1584,7 +1584,13 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } -/** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ +bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +{ + return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), fOverrideMempoolLimit, nAbsurdFee); +} + +/** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { CBlockIndex *pindexSlow = NULL; @@ -2768,10 +2774,9 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara std::vector<uint256> vHashUpdate; BOOST_FOREACH(const CTransaction &tx, block.vtx) { // ignore validation errors in resurrected transactions - list<CTransaction> removed; CValidationState stateDummy; if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { - mempool.removeRecursive(tx, removed); + mempool.removeRecursive(tx); } else if (mempool.exists(tx.GetHash())) { vHashUpdate.push_back(tx.GetHash()); } @@ -2804,7 +2809,7 @@ static int64_t nTimePostConnect = 0; * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock, std::list<CTransaction> &txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>> &txChanged) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock, std::vector<std::shared_ptr<const CTransaction>> &txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>> &txChanged) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2840,7 +2845,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; - mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); + mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, &txConflicted, !IsInitialBlockDownload()); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); @@ -2927,7 +2932,7 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound, std::list<CTransaction>& txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>>& txChanged) +static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound, std::vector<std::shared_ptr<const CTransaction>>& txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>>& txChanged) { AssertLockHeld(cs_main); const CBlockIndex *pindexOldTip = chainActive.Tip(); @@ -3006,9 +3011,8 @@ static void NotifyHeaderTip() { CBlockIndex* pindexHeader = NULL; { LOCK(cs_main); - if (!setBlockIndexCandidates.empty()) { - pindexHeader = *setBlockIndexCandidates.rbegin(); - } + pindexHeader = pindexBestHeader; + if (pindexHeader != pindexHeaderOld) { fNotify = true; fInitialBlockDownload = IsInitialBlockDownload(); @@ -3039,7 +3043,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, break; const CBlockIndex *pindexFork; - std::list<CTransaction> txConflicted; + std::vector<std::shared_ptr<const CTransaction>> txConflicted; bool fInitialDownload; { LOCK(cs_main); @@ -3070,9 +3074,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, // throw all transactions though the signal-interface // while _not_ holding the cs_main lock - BOOST_FOREACH(const CTransaction &tx, txConflicted) + for(std::shared_ptr<const CTransaction> tx : txConflicted) { - GetMainSignals().SyncTransaction(tx, pindexNewTip, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + GetMainSignals().SyncTransaction(*tx, pindexNewTip, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); } // ... and about transactions that got confirmed: for(unsigned int i = 0; i < txChanged.size(); i++) @@ -3968,9 +3972,8 @@ CBlockIndex * InsertBlockIndex(uint256 hash) return pindexNew; } -bool static LoadBlockIndexDB() +bool static LoadBlockIndexDB(const CChainParams& chainparams) { - const CChainParams& chainparams = Params(); if (!pblocktree->LoadBlockIndexGuts(InsertBlockIndex)) return false; @@ -4299,10 +4302,10 @@ void UnloadBlockIndex() fHavePruned = false; } -bool LoadBlockIndex() +bool LoadBlockIndex(const CChainParams& chainparams) { // Load block index from databases - if (!fReindex && !LoadBlockIndexDB()) + if (!fReindex && !LoadBlockIndexDB(chainparams)) return false; return true; } @@ -5443,6 +5446,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, BlockTransactionsRequest req; vRecv >> req; + LOCK(cs_main); + BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); @@ -5829,29 +5834,34 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, BlockTransactions resp; vRecv >> resp; - LOCK(cs_main); + CBlock block; + bool fBlockRead = false; + { + LOCK(cs_main); - map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); - if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || - it->second.first != pfrom->GetId()) { - LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); - return true; - } + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); + if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || + it->second.first != pfrom->GetId()) { + LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + return true; + } - PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; - CBlock block; - ReadStatus status = partialBlock.FillBlock(block, resp.txn); - if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist - Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); - return true; - } else if (status == READ_STATUS_FAILED) { - // Might have collided, fall back to getdata now :( - std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); - pfrom->PushMessage(NetMsgType::GETDATA, invs); - } else { + PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + ReadStatus status = partialBlock.FillBlock(block, resp.txn); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Might have collided, fall back to getdata now :( + std::vector<CInv> invs; + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); + pfrom->PushMessage(NetMsgType::GETDATA, invs); + } else + fBlockRead = true; + } // Don't hold cs_main when we call into ProcessNewBlock + if (fBlockRead) { CValidationState state; ProcessNewBlock(state, chainparams, pfrom, &block, false, NULL); int nDoS; @@ -6924,6 +6934,119 @@ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::D return VersionBitsStateSinceHeight(chainActive.Tip(), params, pos, versionbitscache); } +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(void) +{ + int64_t nExpiryTimeout = GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; + FILE* filestr = fopen((GetDataDir() / "mempool.dat").string().c_str(), "r"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t skipped = 0; + int64_t failed = 0; + int64_t nNow = GetTime(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + double prioritydummy = 0; + while (num--) { + CTransaction tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + mempool.PrioritiseTransaction(tx.GetHash(), tx.GetHash().ToString(), prioritydummy, amountdelta); + } + CValidationState state; + if (nTime + nExpiryTimeout > nNow) { + LOCK(cs_main); + AcceptToMemoryPoolWithTime(mempool, state, tx, true, NULL, nTime); + if (state.IsValid()) { + ++count; + } else { + ++failed; + } + } else { + ++skipped; + } + } + std::map<uint256, CAmount> mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + mempool.PrioritiseTransaction(i.first, i.first.ToString(), prioritydummy, i.second); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i successes, %i failed, %i expired\n", count, failed, skipped); + return true; +} + +void DumpMempool(void) +{ + int64_t start = GetTimeMicros(); + + std::map<uint256, CAmount> mapDeltas; + std::vector<TxMempoolInfo> vinfo; + + { + LOCK(mempool.cs); + for (const auto &i : mempool.mapDeltas) { + mapDeltas[i.first] = i.second.first; + } + vinfo = mempool.infoAll(); + } + + int64_t mid = GetTimeMicros(); + + try { + FILE* filestr = fopen((GetDataDir() / "mempool.dat.new").string().c_str(), "w"); + if (!filestr) { + return; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << (int64_t)i.nTime; + file << (int64_t)i.nFeeDelta; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + FileCommit(file.Get()); + file.fclose(); + RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat"); + int64_t last = GetTimeMicros(); + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*0.000001, (last-mid)*0.000001); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + } +} + class CMainCleanup { public: diff --git a/src/main.h b/src/main.h index 3eab9b89da..35060e34ab 100644 --- a/src/main.h +++ b/src/main.h @@ -237,7 +237,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB /** Initialize a new block tree database + block data on disk */ bool InitBlockIndex(const CChainParams& chainparams); /** Load the block tree and coins database from disk */ -bool LoadBlockIndex(); +bool LoadBlockIndex(const CChainParams& chainparams); /** Unload database information */ void UnloadBlockIndex(); /** Run an instance of the script checking thread */ @@ -291,6 +291,10 @@ void PruneAndFlush(); bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, bool* pfMissingInputs, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0); +/** (try to) add transaction to memory pool with a specified acceptance time **/ +bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0); + /** Convert CValidationState to a human-readable message for logging */ std::string FormatStateMessage(const CValidationState &state); @@ -529,6 +533,12 @@ static const unsigned int REJECT_ALREADY_KNOWN = 0x101; /** Transaction conflicts with a transaction already known */ static const unsigned int REJECT_CONFLICT = 0x102; +/** Dump the mempool to disk. */ +void DumpMempool(); + +/** Load the mempool from disk. */ +bool LoadMempool(); + // The following things handle network-processing logic // (and should be moved to a separate file) diff --git a/src/net.cpp b/src/net.cpp index 1bca168d1d..48ba9588d9 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -970,7 +970,6 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { CAddress addr; int nInbound = 0; int nMaxInbound = nMaxConnections - (nMaxOutbound + nMaxFeeler); - assert(nMaxInbound > 0); if (hSocket != INVALID_SOCKET) if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) @@ -2138,8 +2137,9 @@ bool CConnman::Start(boost::thread_group& threadGroup, CScheduler& scheduler, st // Initiate outbound connections from -addnode threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "addcon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenAddedConnections, this)))); - // Initiate outbound connections - threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "opencon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenConnections, this)))); + // Initiate outbound connections unless connect=0 + if (!mapArgs.count("-connect") || mapMultiArgs["-connect"].size() != 1 || mapMultiArgs["-connect"][0] != "0") + threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "opencon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenConnections, this)))); // Process messages threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "msghand", boost::function<void()>(boost::bind(&CConnman::ThreadMessageHandler, this)))); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index af767aa6c6..ee5102c4f9 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -717,13 +717,10 @@ void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVer { if (modalOverlay) { - if (header) { - /* use clientmodels getHeaderTipHeight and getHeaderTipTime because the NotifyHeaderTip signal does not fire when updating the best header */ - modalOverlay->setKnownBestHeight(clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(clientModel->getHeaderTipTime())); - } - else { + if (header) + modalOverlay->setKnownBestHeight(count, blockDate); + else modalOverlay->tipUpdate(count, blockDate, nVerificationProgress); - } } if (!clientModel) return; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 86fd4ebd65..1a1671f0ee 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -13,7 +13,7 @@ #include "txmempool.h" #include "walletmodel.h" -#include "coincontrol.h" +#include "wallet/coincontrol.h" #include "init.h" #include "main.h" // For minRelayTxFee #include "wallet/wallet.h" diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index 73223735f5..27998f90c5 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -219,7 +219,7 @@ QLabel { color: rgb(40,40,40); }</string> <item row="0" column="1"> <widget class="QLabel" name="numberOfBlocksLeft"> <property name="text"> - <string>unknown...</string> + <string>Unknown...</string> </property> </widget> </item> @@ -245,7 +245,7 @@ QLabel { color: rgb(40,40,40); }</string> </sizepolicy> </property> <property name="text"> - <string>unknown...</string> + <string>Unknown...</string> </property> </widget> </item> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 06e09074d1..33db9f8938 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -411,7 +411,7 @@ </property> </widget> </item> - </layout> + </layout> </item> <item> <layout class="QFormLayout" name="formLayoutCoinControl4"> @@ -1031,7 +1031,7 @@ <item> <widget class="QLabel" name="labelSmartFee3"> <property name="text"> - <string>Confirmation time:</string> + <string>Confirmation time target:</string> </property> <property name="margin"> <number>2</number> @@ -1096,6 +1096,26 @@ </widget> </item> <item> + <spacer name="horizontalSpacer_7"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QLabel" name="confirmationTargetLabel"> + <property name="text"> + <string>(count)</string> + </property> + </widget> + </item> + <item> <spacer name="horizontalSpacer_3"> <property name="orientation"> <enum>Qt::Horizontal</enum> diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index ae0d8f5f63..1a843a07ac 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -132,7 +132,8 @@ void ModalOverlay::tipUpdate(int count, const QDateTime& blockDate, double nVeri if (estimateNumHeadersLeft < 24 && hasBestHeader) { ui->numberOfBlocksLeft->setText(QString::number(bestHeaderHeight - count)); } else { - ui->expectedTimeLeft->setText(tr("Unknown. Syncing Headers...")); + ui->numberOfBlocksLeft->setText(tr("Unknown. Syncing Headers (%1)...").arg(bestHeaderHeight)); + ui->expectedTimeLeft->setText(tr("Unknown...")); } } diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 4b2ba7d624..57b2179435 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -16,7 +16,7 @@ #include "walletmodel.h" #include "base58.h" -#include "coincontrol.h" +#include "wallet/coincontrol.h" #include "main.h" // mempool and minRelayTxFee #include "ui_interface.h" #include "txmempool.h" @@ -110,7 +110,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupCustomFee->setId(ui->radioCustomPerKilobyte, 0); ui->groupCustomFee->setId(ui->radioCustomAtLeast, 1); ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); - ui->sliderSmartFee->setValue(settings.value("nSmartFeeSliderPosition").toInt()); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); minimizeFeeSection(settings.value("fFeeSectionMinimized").toBool()); @@ -172,6 +171,13 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateMinFeeLabel(); updateSmartFeeLabel(); updateGlobalFeeVariables(); + + // set the smartfee-sliders default value (wallets default conf.target or last stored value) + QSettings settings; + if (settings.value("nSmartFeeSliderPosition").toInt() == 0) + ui->sliderSmartFee->setValue(ui->sliderSmartFee->maximum() - model->getDefaultConfirmTarget() + 1); + else + ui->sliderSmartFee->setValue(settings.value("nSmartFeeSliderPosition").toInt()); } } @@ -229,10 +235,17 @@ void SendCoinsDialog::on_sendButton_clicked() // prepare transaction for getting txFee earlier WalletModelTransaction currentTransaction(recipients); WalletModel::SendCoinsReturn prepareStatus; - if (model->getOptionsModel()->getCoinControlFeatures()) // coin control enabled - prepareStatus = model->prepareTransaction(currentTransaction, CoinControlDialog::coinControl); + + // Always use a CCoinControl instance, use the CoinControlDialog instance if CoinControl has been enabled + CCoinControl ctrl; + if (model->getOptionsModel()->getCoinControlFeatures()) + ctrl = *CoinControlDialog::coinControl; + if (ui->radioSmartFee->isChecked()) + ctrl.nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; else - prepareStatus = model->prepareTransaction(currentTransaction); + ctrl.nConfirmTarget = 0; + + prepareStatus = model->prepareTransaction(currentTransaction, &ctrl); // process prepareStatus and on error generate message shown to user processSendCoinsReturn(prepareStatus, @@ -521,7 +534,7 @@ void SendCoinsDialog::processSendCoinsReturn(const WalletModel::SendCoinsReturn msgParams.second = CClientUIInterface::MSG_ERROR; break; case WalletModel::TransactionCommitFailed: - msgParams.first = tr("The transaction was rejected! This might happen if some of the coins in your wallet were already spent, such as if you used a copy of wallet.dat and coins were spent in the copy but not marked as spent here."); + msgParams.first = tr("The transaction was rejected with the following reason: %1").arg(sendCoinsReturn.reasonCommitFailed); msgParams.second = CClientUIInterface::MSG_ERROR; break; case WalletModel::AbsurdFee: @@ -576,6 +589,7 @@ void SendCoinsDialog::updateFeeSectionControls() ui->labelFeeEstimation ->setEnabled(ui->radioSmartFee->isChecked()); ui->labelSmartFeeNormal ->setEnabled(ui->radioSmartFee->isChecked()); ui->labelSmartFeeFast ->setEnabled(ui->radioSmartFee->isChecked()); + ui->confirmationTargetLabel ->setEnabled(ui->radioSmartFee->isChecked()); ui->checkBoxMinimumFee ->setEnabled(ui->radioCustomFee->isChecked()); ui->labelMinFeeWarning ->setEnabled(ui->radioCustomFee->isChecked()); ui->radioCustomPerKilobyte ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); @@ -587,15 +601,17 @@ void SendCoinsDialog::updateGlobalFeeVariables() { if (ui->radioSmartFee->isChecked()) { - nTxConfirmTarget = defaultConfirmTarget - ui->sliderSmartFee->value(); + int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; payTxFee = CFeeRate(0); // set nMinimumTotalFee to 0 to not accidentally pay a custom fee CoinControlDialog::coinControl->nMinimumTotalFee = 0; + + // show the estimated reuquired time for confirmation + ui->confirmationTargetLabel->setText(GUIUtil::formatDurationStr(nConfirmTarget*600)+" / "+tr("%n block(s)", "", nConfirmTarget)); } else { - nTxConfirmTarget = defaultConfirmTarget; payTxFee = CFeeRate(ui->customFee->value()); // if user has selected to set a minimum absolute fee, pass the value to coincontrol @@ -630,7 +646,7 @@ void SendCoinsDialog::updateSmartFeeLabel() if(!model || !model->getOptionsModel()) return; - int nBlocksToConfirm = defaultConfirmTarget - ui->sliderSmartFee->value(); + int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; int estimateFoundAtBlocks = nBlocksToConfirm; CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); if (feeRate <= CFeeRate(0)) // not enough data => minfee @@ -701,6 +717,8 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) if (!checked && model) // coin control features disabled CoinControlDialog::coinControl->SetNull(); + // make sure we set back the confirmation target + updateGlobalFeeVariables(); coinControlUpdateLabels(); } diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 83dac0bd11..b0df495a98 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -26,8 +26,6 @@ QT_BEGIN_NAMESPACE class QUrl; QT_END_NAMESPACE -const int defaultConfirmTarget = 25; - /** Dialog for sending bitcoins */ class SendCoinsDialog : public QDialog { diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 305cb4fefa..3490d1c1cc 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -5,6 +5,7 @@ #include "walletmodel.h" #include "addresstablemodel.h" +#include "consensus/validation.h" #include "guiconstants.h" #include "guiutil.h" #include "paymentserver.h" @@ -328,8 +329,9 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran } CReserveKey *keyChange = transaction.getPossibleKeyChange(); - if(!wallet->CommitTransaction(*newTx, *keyChange, g_connman.get())) - return TransactionCommitFailed; + CValidationState state; + if(!wallet->CommitTransaction(*newTx, *keyChange, g_connman.get(), state)) + return SendCoinsReturn(TransactionCommitFailed, QString::fromStdString(state.GetRejectReason())); CTransaction* t = (CTransaction*)newTx; CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); @@ -698,3 +700,8 @@ bool WalletModel::hdEnabled() const { return wallet->IsHDEnabled(); } + +int WalletModel::getDefaultConfirmTarget() const +{ + return nTxConfirmTarget; +} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index cdac60da36..6a5670e378 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -144,9 +144,13 @@ public: // Return status record for SendCoins, contains error id + information struct SendCoinsReturn { - SendCoinsReturn(StatusCode _status = OK): - status(_status) {} + SendCoinsReturn(StatusCode _status = OK, QString _reasonCommitFailed = "") + : status(_status), + reasonCommitFailed(_reasonCommitFailed) + { + } StatusCode status; + QString reasonCommitFailed; }; // prepare transaction for getting txfee before sending coins @@ -207,6 +211,8 @@ public: bool hdEnabled() const; + int getDefaultConfirmTarget() const; + private: CWallet *wallet; bool fHaveWatchOnly; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f538ddcc04..8caea14adb 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -151,7 +151,7 @@ UniValue getblockcount(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 0) throw runtime_error( "getblockcount\n" - "\nReturns the number of blocks in the longest block chain.\n" + "\nReturns the number of blocks in the longest blockchain.\n" "\nResult:\n" "n (numeric) The current block count\n" "\nExamples:\n" @@ -168,7 +168,7 @@ UniValue getbestblockhash(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 0) throw runtime_error( "getbestblockhash\n" - "\nReturns the hash of the best (tip) block in the longest block chain.\n" + "\nReturns the hash of the best (tip) block in the longest blockchain.\n" "\nResult\n" "\"hex\" (string) the block hash hex encoded\n" "\nExamples\n" @@ -863,7 +863,7 @@ UniValue gettxout(const JSONRPCRequest& request) "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" "2. n (numeric, required) vout number\n" - "3. includemempool (boolean, optional) Whether to include the mem pool\n" + "3. includemempool (boolean, optional) Whether to include the mempool\n" "\nResult:\n" "{\n" " \"bestblock\" : \"hash\", (string) the block hash\n" @@ -1027,7 +1027,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 0) throw runtime_error( "getblockchaininfo\n" - "Returns an object containing various state info regarding block chain processing.\n" + "Returns an object containing various state info regarding blockchain processing.\n" "\nResult:\n" "{\n" " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c14d9d6747..8370a0f43e 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -95,6 +95,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importaddress", 2 }, { "importaddress", 3 }, { "importpubkey", 2 }, + { "importmulti", 0 }, + { "importmulti", 1 }, { "verifychain", 0 }, { "verifychain", 1 }, { "keypoolrefill", 0 }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index d509dd691f..be0776ea22 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -230,7 +230,7 @@ UniValue getmininginfo(const JSONRPCRequest& request) " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" " \"errors\": \"...\" (string) Current errors\n" " \"networkhashps\": nnn, (numeric) The network hashes per second\n" - " \"pooledtx\": n (numeric) The size of the mem pool\n" + " \"pooledtx\": n (numeric) The size of the mempool\n" " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" "}\n" "\nExamples:\n" diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index eaef4856b3..3193985803 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -57,7 +57,7 @@ UniValue getinfo(const JSONRPCRequest& request) " \"proxy\": \"host:port\", (string, optional) the proxy used by the server\n" " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"testnet\": true|false, (boolean) if the server is using testnet or not\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since GMT epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee set in " + CURRENCY_UNIT + "/kB\n" @@ -450,10 +450,53 @@ UniValue setmocktime(const JSONRPCRequest& request) return NullUniValue; } +static UniValue RPCLockedMemoryInfo() +{ + LockedPool::Stats stats = LockedPoolManager::Instance().stats(); + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("used", uint64_t(stats.used))); + obj.push_back(Pair("free", uint64_t(stats.free))); + obj.push_back(Pair("total", uint64_t(stats.total))); + obj.push_back(Pair("locked", uint64_t(stats.locked))); + obj.push_back(Pair("chunks_used", uint64_t(stats.chunks_used))); + obj.push_back(Pair("chunks_free", uint64_t(stats.chunks_free))); + return obj; +} + +UniValue getmemoryinfo(const JSONRPCRequest& request) +{ + /* Please, avoid using the word "pool" here in the RPC interface or help, + * as users will undoubtedly confuse it with the other "memory pool" + */ + if (request.fHelp || request.params.size() != 0) + throw runtime_error( + "getmemoryinfo\n" + "Returns an object containing information about memory usage.\n" + "\nResult:\n" + "{\n" + " \"locked\": { (json object) Information about locked memory manager\n" + " \"used\": xxxxx, (numeric) Number of bytes used\n" + " \"free\": xxxxx, (numeric) Number of bytes available in current arenas\n" + " \"total\": xxxxxxx, (numeric) Total number of bytes managed\n" + " \"locked\": xxxxxx, (numeric) Amount of bytes that succeeded locking. If this number is smaller than total, locking pages failed at some point and key data could be swapped to disk.\n" + " \"chunks_used\": xxxxx, (numeric) Number allocated chunks\n" + " \"chunks_free\": xxxxx, (numeric) Number unused chunks\n" + " }\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getmemoryinfo", "") + + HelpExampleRpc("getmemoryinfo", "") + ); + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("locked", RPCLockedMemoryInfo())); + return obj; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- { "control", "getinfo", &getinfo, true }, /* uses wallet if enabled */ + { "control", "getmemoryinfo", &getmemoryinfo, true }, { "util", "validateaddress", &validateaddress, true }, /* uses wallet if enabled */ { "util", "createmultisig", &createmultisig, true }, { "util", "verifymessage", &verifymessage, true }, diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 29d0bee1b2..164e0f00e2 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -147,6 +147,8 @@ uint256 ParseHashV(const UniValue& v, string strName) strHex = v.get_str(); if (!IsHex(strHex)) // Note: IsHex("") is false throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + if (64 != strHex.length()) + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d)", strName, 64, strHex.length())); uint256 result; result.SetHex(strHex); return result; diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 1ec40fe830..67064314ef 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -6,7 +6,8 @@ #ifndef BITCOIN_SUPPORT_ALLOCATORS_SECURE_H #define BITCOIN_SUPPORT_ALLOCATORS_SECURE_H -#include "support/pagelocker.h" +#include "support/lockedpool.h" +#include "support/cleanse.h" #include <string> @@ -39,20 +40,15 @@ struct secure_allocator : public std::allocator<T> { T* allocate(std::size_t n, const void* hint = 0) { - T* p; - p = std::allocator<T>::allocate(n, hint); - if (p != NULL) - LockedPageManager::Instance().LockRange(p, sizeof(T) * n); - return p; + return static_cast<T*>(LockedPoolManager::Instance().alloc(sizeof(T) * n)); } void deallocate(T* p, std::size_t n) { if (p != NULL) { memory_cleanse(p, sizeof(T) * n); - LockedPageManager::Instance().UnlockRange(p, sizeof(T) * n); } - std::allocator<T>::deallocate(p, n); + LockedPoolManager::Instance().free(p); } }; diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp new file mode 100644 index 0000000000..813869a131 --- /dev/null +++ b/src/support/lockedpool.cpp @@ -0,0 +1,390 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "support/lockedpool.h" +#include "support/cleanse.h" + +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif + +#ifdef WIN32 +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0501 +#define WIN32_LEAN_AND_MEAN 1 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include <windows.h> +#else +#include <sys/mman.h> // for mmap +#include <sys/resource.h> // for getrlimit +#include <limits.h> // for PAGESIZE +#include <unistd.h> // for sysconf +#endif + +LockedPoolManager* LockedPoolManager::_instance = NULL; +std::once_flag LockedPoolManager::init_flag; + +/*******************************************************************************/ +// Utilities +// +/** Align up to power of 2 */ +static inline size_t align_up(size_t x, size_t align) +{ + return (x + align - 1) & ~(align - 1); +} + +/*******************************************************************************/ +// Implementation: Arena + +Arena::Arena(void *base_in, size_t size_in, size_t alignment_in): + base(static_cast<char*>(base_in)), end(static_cast<char*>(base_in) + size_in), alignment(alignment_in) +{ + // Start with one free chunk that covers the entire arena + chunks.emplace(base, Chunk(size_in, false)); +} + +Arena::~Arena() +{ +} + +void* Arena::alloc(size_t size) +{ + // Round to next multiple of alignment + size = align_up(size, alignment); + + // Don't handle zero-sized chunks, or those bigger than MAX_SIZE + if (size == 0 || size >= Chunk::MAX_SIZE) { + return nullptr; + } + + for (auto& chunk: chunks) { + if (!chunk.second.isInUse() && size <= chunk.second.getSize()) { + char* _base = chunk.first; + size_t leftover = chunk.second.getSize() - size; + if (leftover > 0) { // Split chunk + chunks.emplace(_base + size, Chunk(leftover, false)); + chunk.second.setSize(size); + } + chunk.second.setInUse(true); + return reinterpret_cast<void*>(_base); + } + } + return nullptr; +} + +void Arena::free(void *ptr) +{ + // Freeing the NULL pointer is OK. + if (ptr == nullptr) { + return; + } + auto i = chunks.find(static_cast<char*>(ptr)); + if (i == chunks.end() || !i->second.isInUse()) { + throw std::runtime_error("Arena: invalid or double free"); + } + + i->second.setInUse(false); + + if (i != chunks.begin()) { // Absorb into previous chunk if exists and free + auto prev = i; + --prev; + if (!prev->second.isInUse()) { + // Absorb current chunk size into previous chunk. + prev->second.setSize(prev->second.getSize() + i->second.getSize()); + // Erase current chunk. Erasing does not invalidate current + // iterators for a map, except for that pointing to the object + // itself, which will be overwritten in the next statement. + chunks.erase(i); + // From here on, the previous chunk is our current chunk. + i = prev; + } + } + auto next = i; + ++next; + if (next != chunks.end()) { // Absorb next chunk if exists and free + if (!next->second.isInUse()) { + // Absurb next chunk size into current chunk + i->second.setSize(i->second.getSize() + next->second.getSize()); + // Erase next chunk. + chunks.erase(next); + } + } +} + +Arena::Stats Arena::stats() const +{ + Arena::Stats r; + r.used = r.free = r.total = r.chunks_used = r.chunks_free = 0; + for (const auto& chunk: chunks) { + if (chunk.second.isInUse()) { + r.used += chunk.second.getSize(); + r.chunks_used += 1; + } else { + r.free += chunk.second.getSize(); + r.chunks_free += 1; + } + r.total += chunk.second.getSize(); + } + return r; +} + +#ifdef ARENA_DEBUG +void Arena::walk() const +{ + for (const auto& chunk: chunks) { + std::cout << + "0x" << std::hex << std::setw(16) << std::setfill('0') << chunk.first << + " 0x" << std::hex << std::setw(16) << std::setfill('0') << chunk.second.getSize() << + " 0x" << chunk.second.isInUse() << std::endl; + } + std::cout << std::endl; +} +#endif + +/*******************************************************************************/ +// Implementation: Win32LockedPageAllocator + +#ifdef WIN32 +/** LockedPageAllocator specialized for Windows. + */ +class Win32LockedPageAllocator: public LockedPageAllocator +{ +public: + Win32LockedPageAllocator(); + void* AllocateLocked(size_t len, bool *lockingSuccess); + void FreeLocked(void* addr, size_t len); + size_t GetLimit(); +private: + size_t page_size; +}; + +Win32LockedPageAllocator::Win32LockedPageAllocator() +{ + // Determine system page size in bytes + SYSTEM_INFO sSysInfo; + GetSystemInfo(&sSysInfo); + page_size = sSysInfo.dwPageSize; +} +void *Win32LockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) +{ + len = align_up(len, page_size); + void *addr = VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + if (addr) { + // VirtualLock is used to attempt to keep keying material out of swap. Note + // that it does not provide this as a guarantee, but, in practice, memory + // that has been VirtualLock'd almost never gets written to the pagefile + // except in rare circumstances where memory is extremely low. + *lockingSuccess = VirtualLock(const_cast<void*>(addr), len) != 0; + } + return addr; +} +void Win32LockedPageAllocator::FreeLocked(void* addr, size_t len) +{ + len = align_up(len, page_size); + memory_cleanse(addr, len); + VirtualUnlock(const_cast<void*>(addr), len); +} + +size_t Win32LockedPageAllocator::GetLimit() +{ + // TODO is there a limit on windows, how to get it? + return std::numeric_limits<size_t>::max(); +} +#endif + +/*******************************************************************************/ +// Implementation: PosixLockedPageAllocator + +#ifndef WIN32 +/** LockedPageAllocator specialized for OSes that don't try to be + * special snowflakes. + */ +class PosixLockedPageAllocator: public LockedPageAllocator +{ +public: + PosixLockedPageAllocator(); + void* AllocateLocked(size_t len, bool *lockingSuccess); + void FreeLocked(void* addr, size_t len); + size_t GetLimit(); +private: + size_t page_size; +}; + +PosixLockedPageAllocator::PosixLockedPageAllocator() +{ + // Determine system page size in bytes +#if defined(PAGESIZE) // defined in limits.h + page_size = PAGESIZE; +#else // assume some POSIX OS + page_size = sysconf(_SC_PAGESIZE); +#endif +} + +// Some systems (at least OS X) do not define MAP_ANONYMOUS yet and define +// MAP_ANON which is deprecated +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) +{ + void *addr; + len = align_up(len, page_size); + addr = mmap(nullptr, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (addr) { + *lockingSuccess = mlock(addr, len) == 0; + } + return addr; +} +void PosixLockedPageAllocator::FreeLocked(void* addr, size_t len) +{ + len = align_up(len, page_size); + memory_cleanse(addr, len); + munlock(addr, len); + munmap(addr, len); +} +size_t PosixLockedPageAllocator::GetLimit() +{ +#ifdef RLIMIT_MEMLOCK + struct rlimit rlim; + if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) { + if (rlim.rlim_cur != RLIM_INFINITY) { + return rlim.rlim_cur; + } + } +#endif + return std::numeric_limits<size_t>::max(); +} +#endif + +/*******************************************************************************/ +// Implementation: LockedPool + +LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in, LockingFailed_Callback lf_cb_in): + allocator(std::move(allocator_in)), lf_cb(lf_cb_in), cumulative_bytes_locked(0) +{ +} + +LockedPool::~LockedPool() +{ +} +void* LockedPool::alloc(size_t size) +{ + std::lock_guard<std::mutex> lock(mutex); + // Try allocating from each current arena + for (auto &arena: arenas) { + void *addr = arena.alloc(size); + if (addr) { + return addr; + } + } + // If that fails, create a new one + if (new_arena(ARENA_SIZE, ARENA_ALIGN)) { + return arenas.back().alloc(size); + } + return nullptr; +} + +void LockedPool::free(void *ptr) +{ + std::lock_guard<std::mutex> lock(mutex); + // TODO we can do better than this linear search by keeping a map of arena + // extents to arena, and looking up the address. + for (auto &arena: arenas) { + if (arena.addressInArena(ptr)) { + arena.free(ptr); + return; + } + } + throw std::runtime_error("LockedPool: invalid address not pointing to any arena"); +} + +LockedPool::Stats LockedPool::stats() const +{ + std::lock_guard<std::mutex> lock(mutex); + LockedPool::Stats r; + r.used = r.free = r.total = r.chunks_used = r.chunks_free = 0; + r.locked = cumulative_bytes_locked; + for (const auto &arena: arenas) { + Arena::Stats i = arena.stats(); + r.used += i.used; + r.free += i.free; + r.total += i.total; + r.chunks_used += i.chunks_used; + r.chunks_free += i.chunks_free; + } + return r; +} + +bool LockedPool::new_arena(size_t size, size_t align) +{ + bool locked; + // If this is the first arena, handle this specially: Cap the upper size + // by the process limit. This makes sure that the first arena will at least + // be locked. An exception to this is if the process limit is 0: + // in this case no memory can be locked at all so we'll skip past this logic. + if (arenas.empty()) { + size_t limit = allocator->GetLimit(); + if (limit > 0) { + size = std::min(size, limit); + } + } + void *addr = allocator->AllocateLocked(size, &locked); + if (!addr) { + return false; + } + if (locked) { + cumulative_bytes_locked += size; + } else if (lf_cb) { // Call the locking-failed callback if locking failed + if (!lf_cb()) { // If the callback returns false, free the memory and fail, otherwise consider the user warned and proceed. + allocator->FreeLocked(addr, size); + return false; + } + } + arenas.emplace_back(allocator.get(), addr, size, align); + return true; +} + +LockedPool::LockedPageArena::LockedPageArena(LockedPageAllocator *allocator_in, void *base_in, size_t size_in, size_t align_in): + Arena(base_in, size_in, align_in), base(base_in), size(size_in), allocator(allocator_in) +{ +} +LockedPool::LockedPageArena::~LockedPageArena() +{ + allocator->FreeLocked(base, size); +} + +/*******************************************************************************/ +// Implementation: LockedPoolManager +// +LockedPoolManager::LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator): + LockedPool(std::move(allocator), &LockedPoolManager::LockingFailed) +{ +} + +bool LockedPoolManager::LockingFailed() +{ + // TODO: log something but how? without including util.h + return true; +} + +void LockedPoolManager::CreateInstance() +{ + // Using a local static instance guarantees that the object is initialized + // when it's first needed and also deinitialized after all objects that use + // it are done with it. I can think of one unlikely scenario where we may + // have a static deinitialization order/problem, but the check in + // LockedPoolManagerBase's destructor helps us detect if that ever happens. +#ifdef WIN32 + std::unique_ptr<LockedPageAllocator> allocator(new Win32LockedPageAllocator()); +#else + std::unique_ptr<LockedPageAllocator> allocator(new PosixLockedPageAllocator()); +#endif + static LockedPoolManager instance(std::move(allocator)); + LockedPoolManager::_instance = &instance; +} diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h new file mode 100644 index 0000000000..526c17a73f --- /dev/null +++ b/src/support/lockedpool.h @@ -0,0 +1,251 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SUPPORT_LOCKEDPOOL_H +#define BITCOIN_SUPPORT_LOCKEDPOOL_H + +#include <stdint.h> +#include <list> +#include <map> +#include <mutex> +#include <memory> + +/** + * OS-dependent allocation and deallocation of locked/pinned memory pages. + * Abstract base class. + */ +class LockedPageAllocator +{ +public: + virtual ~LockedPageAllocator() {} + /** Allocate and lock memory pages. + * If len is not a multiple of the system page size, it is rounded up. + * Returns 0 in case of allocation failure. + * + * If locking the memory pages could not be accomplished it will still + * return the memory, however the lockingSuccess flag will be false. + * lockingSuccess is undefined if the allocation fails. + */ + virtual void* AllocateLocked(size_t len, bool *lockingSuccess) = 0; + + /** Unlock and free memory pages. + * Clear the memory before unlocking. + */ + virtual void FreeLocked(void* addr, size_t len) = 0; + + /** Get the total limit on the amount of memory that may be locked by this + * process, in bytes. Return size_t max if there is no limit or the limit + * is unknown. Return 0 if no memory can be locked at all. + */ + virtual size_t GetLimit() = 0; +}; + +/* An arena manages a contiguous region of memory by dividing it into + * chunks. + */ +class Arena +{ +public: + Arena(void *base, size_t size, size_t alignment); + virtual ~Arena(); + + /** A chunk of memory. + */ + struct Chunk + { + /** Most significant bit of size_t. This is used to mark + * in-usedness of chunk. + */ + const static size_t SIZE_MSB = 1LLU << ((sizeof(size_t)*8)-1); + /** Maximum size of a chunk */ + const static size_t MAX_SIZE = SIZE_MSB - 1; + + Chunk(size_t size_in, bool used_in): + size(size_in | (used_in ? SIZE_MSB : 0)) {} + + bool isInUse() const { return size & SIZE_MSB; } + void setInUse(bool used_in) { size = (size & ~SIZE_MSB) | (used_in ? SIZE_MSB : 0); } + size_t getSize() const { return size & ~SIZE_MSB; } + void setSize(size_t size_in) { size = (size & SIZE_MSB) | size_in; } + private: + size_t size; + }; + /** Memory statistics. */ + struct Stats + { + size_t used; + size_t free; + size_t total; + size_t chunks_used; + size_t chunks_free; + }; + + /** Allocate size bytes from this arena. + * Returns pointer on success, or 0 if memory is full or + * the application tried to allocate 0 bytes. + */ + void* alloc(size_t size); + + /** Free a previously allocated chunk of memory. + * Freeing the zero pointer has no effect. + * Raises std::runtime_error in case of error. + */ + void free(void *ptr); + + /** Get arena usage statistics */ + Stats stats() const; + +#ifdef ARENA_DEBUG + void walk() const; +#endif + + /** Return whether a pointer points inside this arena. + * This returns base <= ptr < (base+size) so only use it for (inclusive) + * chunk starting addresses. + */ + bool addressInArena(void *ptr) const { return ptr >= base && ptr < end; } +private: + Arena(const Arena& other) = delete; // non construction-copyable + Arena& operator=(const Arena&) = delete; // non copyable + + /** Map of chunk address to chunk information. This class makes use of the + * sorted order to merge previous and next chunks during deallocation. + */ + std::map<char*, Chunk> chunks; + /** Base address of arena */ + char* base; + /** End address of arena */ + char* end; + /** Minimum chunk alignment */ + size_t alignment; +}; + +/** Pool for locked memory chunks. + * + * To avoid sensitive key data from being swapped to disk, the memory in this pool + * is locked/pinned. + * + * An arena manages a contiguous region of memory. The pool starts out with one arena + * but can grow to multiple arenas if the need arises. + * + * Unlike a normal C heap, the administrative structures are seperate from the managed + * memory. This has been done as the sizes and bases of objects are not in themselves sensitive + * information, as to conserve precious locked memory. In some operating systems + * the amount of memory that can be locked is small. + */ +class LockedPool +{ +public: + /** Size of one arena of locked memory. This is a compromise. + * Do not set this too low, as managing many arenas will increase + * allocation and deallocation overhead. Setting it too high allocates + * more locked memory from the OS than strictly necessary. + */ + static const size_t ARENA_SIZE = 256*1024; + /** Chunk alignment. Another compromise. Setting this too high will waste + * memory, setting it too low will facilitate fragmentation. + */ + static const size_t ARENA_ALIGN = 16; + + /** Callback when allocation succeeds but locking fails. + */ + typedef bool (*LockingFailed_Callback)(); + + /** Memory statistics. */ + struct Stats + { + size_t used; + size_t free; + size_t total; + size_t locked; + size_t chunks_used; + size_t chunks_free; + }; + + /** Create a new LockedPool. This takes ownership of the MemoryPageLocker, + * you can only instantiate this with LockedPool(std::move(...)). + * + * The second argument is an optional callback when locking a newly allocated arena failed. + * If this callback is provided and returns false, the allocation fails (hard fail), if + * it returns true the allocation proceeds, but it could warn. + */ + LockedPool(std::unique_ptr<LockedPageAllocator> allocator, LockingFailed_Callback lf_cb_in = 0); + ~LockedPool(); + + /** Allocate size bytes from this arena. + * Returns pointer on success, or 0 if memory is full or + * the application tried to allocate 0 bytes. + */ + void* alloc(size_t size); + + /** Free a previously allocated chunk of memory. + * Freeing the zero pointer has no effect. + * Raises std::runtime_error in case of error. + */ + void free(void *ptr); + + /** Get pool usage statistics */ + Stats stats() const; +private: + LockedPool(const LockedPool& other) = delete; // non construction-copyable + LockedPool& operator=(const LockedPool&) = delete; // non copyable + + std::unique_ptr<LockedPageAllocator> allocator; + + /** Create an arena from locked pages */ + class LockedPageArena: public Arena + { + public: + LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align); + ~LockedPageArena(); + private: + void *base; + size_t size; + LockedPageAllocator *allocator; + }; + + bool new_arena(size_t size, size_t align); + + std::list<LockedPageArena> arenas; + LockingFailed_Callback lf_cb; + size_t cumulative_bytes_locked; + /** Mutex protects access to this pool's data structures, including arenas. + */ + mutable std::mutex mutex; +}; + +/** + * Singleton class to keep track of locked (ie, non-swappable) memory, for use in + * std::allocator templates. + * + * Some implementations of the STL allocate memory in some constructors (i.e., see + * MSVC's vector<T> implementation where it allocates 1 byte of memory in the allocator.) + * Due to the unpredictable order of static initializers, we have to make sure the + * LockedPoolManager instance exists before any other STL-based objects that use + * secure_allocator are created. So instead of having LockedPoolManager also be + * static-initialized, it is created on demand. + */ +class LockedPoolManager : public LockedPool +{ +public: + /** Return the current instance, or create it once */ + static LockedPoolManager& Instance() + { + std::call_once(LockedPoolManager::init_flag, LockedPoolManager::CreateInstance); + return *LockedPoolManager::_instance; + } + +private: + LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator); + + /** Create a new LockedPoolManager specialized to the OS */ + static void CreateInstance(); + /** Called when locking fails, warn the user here */ + static bool LockingFailed(); + + static LockedPoolManager* _instance; + static std::once_flag init_flag; +}; + +#endif // BITCOIN_SUPPORT_LOCKEDPOOL_H diff --git a/src/support/pagelocker.cpp b/src/support/pagelocker.cpp deleted file mode 100644 index 7cea2d88c5..0000000000 --- a/src/support/pagelocker.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2009-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "support/pagelocker.h" - -#if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" -#endif - -#ifdef WIN32 -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0501 -#define WIN32_LEAN_AND_MEAN 1 -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include <windows.h> -// This is used to attempt to keep keying material out of swap -// Note that VirtualLock does not provide this as a guarantee on Windows, -// but, in practice, memory that has been VirtualLock'd almost never gets written to -// the pagefile except in rare circumstances where memory is extremely low. -#else -#include <sys/mman.h> -#include <limits.h> // for PAGESIZE -#include <unistd.h> // for sysconf -#endif - -LockedPageManager* LockedPageManager::_instance = NULL; -boost::once_flag LockedPageManager::init_flag = BOOST_ONCE_INIT; - -/** Determine system page size in bytes */ -static inline size_t GetSystemPageSize() -{ - size_t page_size; -#if defined(WIN32) - SYSTEM_INFO sSysInfo; - GetSystemInfo(&sSysInfo); - page_size = sSysInfo.dwPageSize; -#elif defined(PAGESIZE) // defined in limits.h - page_size = PAGESIZE; -#else // assume some POSIX OS - page_size = sysconf(_SC_PAGESIZE); -#endif - return page_size; -} - -bool MemoryPageLocker::Lock(const void* addr, size_t len) -{ -#ifdef WIN32 - return VirtualLock(const_cast<void*>(addr), len) != 0; -#else - return mlock(addr, len) == 0; -#endif -} - -bool MemoryPageLocker::Unlock(const void* addr, size_t len) -{ -#ifdef WIN32 - return VirtualUnlock(const_cast<void*>(addr), len) != 0; -#else - return munlock(addr, len) == 0; -#endif -} - -LockedPageManager::LockedPageManager() : LockedPageManagerBase<MemoryPageLocker>(GetSystemPageSize()) -{ -} diff --git a/src/support/pagelocker.h b/src/support/pagelocker.h deleted file mode 100644 index 538bf39453..0000000000 --- a/src/support/pagelocker.h +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_SUPPORT_PAGELOCKER_H -#define BITCOIN_SUPPORT_PAGELOCKER_H - -#include "support/cleanse.h" - -#include <map> - -#include <boost/thread/mutex.hpp> -#include <boost/thread/once.hpp> - -/** - * Thread-safe class to keep track of locked (ie, non-swappable) memory pages. - * - * Memory locks do not stack, that is, pages which have been locked several times by calls to mlock() - * will be unlocked by a single call to munlock(). This can result in keying material ending up in swap when - * those functions are used naively. This class simulates stacking memory locks by keeping a counter per page. - * - * @note By using a map from each page base address to lock count, this class is optimized for - * small objects that span up to a few pages, mostly smaller than a page. To support large allocations, - * something like an interval tree would be the preferred data structure. - */ -template <class Locker> -class LockedPageManagerBase -{ -public: - LockedPageManagerBase(size_t _page_size) : page_size(_page_size) - { - // Determine bitmask for extracting page from address - assert(!(_page_size & (_page_size - 1))); // size must be power of two - page_mask = ~(_page_size - 1); - } - - ~LockedPageManagerBase() - { - } - - - // For all pages in affected range, increase lock count - void LockRange(void* p, size_t size) - { - boost::mutex::scoped_lock lock(mutex); - if (!size) - return; - const size_t base_addr = reinterpret_cast<size_t>(p); - const size_t start_page = base_addr & page_mask; - const size_t end_page = (base_addr + size - 1) & page_mask; - for (size_t page = start_page; page <= end_page; page += page_size) { - Histogram::iterator it = histogram.find(page); - if (it == histogram.end()) // Newly locked page - { - locker.Lock(reinterpret_cast<void*>(page), page_size); - histogram.insert(std::make_pair(page, 1)); - } else // Page was already locked; increase counter - { - it->second += 1; - } - } - } - - // For all pages in affected range, decrease lock count - void UnlockRange(void* p, size_t size) - { - boost::mutex::scoped_lock lock(mutex); - if (!size) - return; - const size_t base_addr = reinterpret_cast<size_t>(p); - const size_t start_page = base_addr & page_mask; - const size_t end_page = (base_addr + size - 1) & page_mask; - for (size_t page = start_page; page <= end_page; page += page_size) { - Histogram::iterator it = histogram.find(page); - assert(it != histogram.end()); // Cannot unlock an area that was not locked - // Decrease counter for page, when it is zero, the page will be unlocked - it->second -= 1; - if (it->second == 0) // Nothing on the page anymore that keeps it locked - { - // Unlock page and remove the count from histogram - locker.Unlock(reinterpret_cast<void*>(page), page_size); - histogram.erase(it); - } - } - } - - // Get number of locked pages for diagnostics - int GetLockedPageCount() - { - boost::mutex::scoped_lock lock(mutex); - return histogram.size(); - } - -private: - Locker locker; - boost::mutex mutex; - size_t page_size, page_mask; - // map of page base address to lock count - typedef std::map<size_t, int> Histogram; - Histogram histogram; -}; - - -/** - * OS-dependent memory page locking/unlocking. - * Defined as policy class to make stubbing for test possible. - */ -class MemoryPageLocker -{ -public: - /** Lock memory pages. - * addr and len must be a multiple of the system page size - */ - bool Lock(const void* addr, size_t len); - /** Unlock memory pages. - * addr and len must be a multiple of the system page size - */ - bool Unlock(const void* addr, size_t len); -}; - -/** - * Singleton class to keep track of locked (ie, non-swappable) memory pages, for use in - * std::allocator templates. - * - * Some implementations of the STL allocate memory in some constructors (i.e., see - * MSVC's vector<T> implementation where it allocates 1 byte of memory in the allocator.) - * Due to the unpredictable order of static initializers, we have to make sure the - * LockedPageManager instance exists before any other STL-based objects that use - * secure_allocator are created. So instead of having LockedPageManager also be - * static-initialized, it is created on demand. - */ -class LockedPageManager : public LockedPageManagerBase<MemoryPageLocker> -{ -public: - static LockedPageManager& Instance() - { - boost::call_once(LockedPageManager::CreateInstance, LockedPageManager::init_flag); - return *LockedPageManager::_instance; - } - -private: - LockedPageManager(); - - static void CreateInstance() - { - // Using a local static instance guarantees that the object is initialized - // when it's first needed and also deinitialized after all objects that use - // it are done with it. I can think of one unlikely scenario where we may - // have a static deinitialization order/problem, but the check in - // LockedPageManagerBase's destructor helps us detect if that ever happens. - static LockedPageManager instance; - LockedPageManager::_instance = &instance; - } - - static LockedPageManager* _instance; - static boost::once_flag init_flag; -}; - -// -// Functions for directly locking/unlocking memory objects. -// Intended for non-dynamically allocated structures. -// -template <typename T> -void LockObject(const T& t) -{ - LockedPageManager::Instance().LockRange((void*)(&t), sizeof(T)); -} - -template <typename T> -void UnlockObject(const T& t) -{ - memory_cleanse((void*)(&t), sizeof(T)); - LockedPageManager::Instance().UnlockRange((void*)(&t), sizeof(T)); -} - -#endif // BITCOIN_SUPPORT_PAGELOCKER_H diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index 613f6c12d7..f0e848655f 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -11,110 +11,214 @@ BOOST_FIXTURE_TEST_SUITE(allocator_tests, BasicTestingSetup) -// Dummy memory page locker for platform independent tests -static const void *last_lock_addr, *last_unlock_addr; -static size_t last_lock_len, last_unlock_len; -class TestLocker +BOOST_AUTO_TEST_CASE(arena_tests) { -public: - bool Lock(const void *addr, size_t len) + // Fake memory base address for testing + // without actually using memory. + void *synth_base = reinterpret_cast<void*>(0x08000000); + const size_t synth_size = 1024*1024; + Arena b(synth_base, synth_size, 16); + void *chunk = b.alloc(1000); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(chunk != nullptr); + BOOST_CHECK(b.stats().used == 1008); // Aligned to 16 + BOOST_CHECK(b.stats().total == synth_size); // Nothing has disappeared? + b.free(chunk); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 0); + BOOST_CHECK(b.stats().free == synth_size); + try { // Test exception on double-free + b.free(chunk); + BOOST_CHECK(0); + } catch(std::runtime_error &) { - last_lock_addr = addr; - last_lock_len = len; - return true; } - bool Unlock(const void *addr, size_t len) - { - last_unlock_addr = addr; - last_unlock_len = len; - return true; + + void *a0 = b.alloc(128); + BOOST_CHECK(a0 == synth_base); // first allocation must start at beginning + void *a1 = b.alloc(256); + void *a2 = b.alloc(512); + BOOST_CHECK(b.stats().used == 896); + BOOST_CHECK(b.stats().total == synth_size); +#ifdef ARENA_DEBUG + b.walk(); +#endif + b.free(a0); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 768); + b.free(a1); + BOOST_CHECK(b.stats().used == 512); + void *a3 = b.alloc(128); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 640); + b.free(a2); + BOOST_CHECK(b.stats().used == 128); + b.free(a3); + BOOST_CHECK(b.stats().used == 0); + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); + + std::vector<void*> addr; + BOOST_CHECK(b.alloc(0) == nullptr); // allocating 0 always returns nullptr +#ifdef ARENA_DEBUG + b.walk(); +#endif + // Sweeping allocate all memory + for (int x=0; x<1024; ++x) + addr.push_back(b.alloc(1024)); + BOOST_CHECK(addr[0] == synth_base); // first allocation must start at beginning + BOOST_CHECK(b.stats().free == 0); + BOOST_CHECK(b.alloc(1024) == nullptr); // memory is full, this must return nullptr + BOOST_CHECK(b.alloc(0) == nullptr); + for (int x=0; x<1024; ++x) + b.free(addr[x]); + addr.clear(); + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); + + // Now in the other direction... + for (int x=0; x<1024; ++x) + addr.push_back(b.alloc(1024)); + for (int x=0; x<1024; ++x) + b.free(addr[1023-x]); + addr.clear(); + + // Now allocate in smaller unequal chunks, then deallocate haphazardly + // Not all the chunks will succeed allocating, but freeing nullptr is + // allowed so that is no problem. + for (int x=0; x<2048; ++x) + addr.push_back(b.alloc(x+1)); + for (int x=0; x<2048; ++x) + b.free(addr[((x*23)%2048)^242]); + addr.clear(); + + // Go entirely wild: free and alloc interleaved, + // generate targets and sizes using pseudo-randomness. + for (int x=0; x<2048; ++x) + addr.push_back(0); + uint32_t s = 0x12345678; + for (int x=0; x<5000; ++x) { + int idx = s & (addr.size()-1); + if (s & 0x80000000) { + b.free(addr[idx]); + addr[idx] = 0; + } else if(!addr[idx]) { + addr[idx] = b.alloc((s >> 16) & 2047); + } + bool lsb = s & 1; + s >>= 1; + if (lsb) + s ^= 0xf00f00f0; // LFSR period 0xf7ffffe0 } -}; + for (void *ptr: addr) + b.free(ptr); + addr.clear(); -BOOST_AUTO_TEST_CASE(test_LockedPageManagerBase) + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); +} + +/** Mock LockedPageAllocator for testing */ +class TestLockedPageAllocator: public LockedPageAllocator { - const size_t test_page_size = 4096; - LockedPageManagerBase<TestLocker> lpm(test_page_size); - size_t addr; - last_lock_addr = last_unlock_addr = 0; - last_lock_len = last_unlock_len = 0; - - /* Try large number of small objects */ - addr = 0; - for(int i=0; i<1000; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), 33); - addr += 33; - } - /* Try small number of page-sized objects, straddling two pages */ - addr = test_page_size*100 + 53; - for(int i=0; i<100; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; - } - /* Try small number of page-sized objects aligned to exactly one page */ - addr = test_page_size*300; - for(int i=0; i<100; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; - } - /* one very large object, straddling pages */ - lpm.LockRange(reinterpret_cast<void*>(test_page_size*600+1), test_page_size*500); - BOOST_CHECK(last_lock_addr == reinterpret_cast<void*>(test_page_size*(600+500))); - /* one very large object, page aligned */ - lpm.LockRange(reinterpret_cast<void*>(test_page_size*1200), test_page_size*500-1); - BOOST_CHECK(last_lock_addr == reinterpret_cast<void*>(test_page_size*(1200+500-1))); - - BOOST_CHECK(lpm.GetLockedPageCount() == ( - (1000*33+test_page_size-1)/test_page_size + // small objects - 101 + 100 + // page-sized objects - 501 + 500)); // large objects - BOOST_CHECK((last_lock_len & (test_page_size-1)) == 0); // always lock entire pages - BOOST_CHECK(last_unlock_len == 0); // nothing unlocked yet - - /* And unlock again */ - addr = 0; - for(int i=0; i<1000; ++i) +public: + TestLockedPageAllocator(int count_in, int lockedcount_in): count(count_in), lockedcount(lockedcount_in) {} + void* AllocateLocked(size_t len, bool *lockingSuccess) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), 33); - addr += 33; + *lockingSuccess = false; + if (count > 0) { + --count; + + if (lockedcount > 0) { + --lockedcount; + *lockingSuccess = true; + } + + return reinterpret_cast<void*>(0x08000000 + (count<<24)); // Fake address, do not actually use this memory + } + return 0; } - addr = test_page_size*100 + 53; - for(int i=0; i<100; ++i) + void FreeLocked(void* addr, size_t len) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; } - addr = test_page_size*300; - for(int i=0; i<100; ++i) + size_t GetLimit() { - lpm.UnlockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; + return std::numeric_limits<size_t>::max(); } - lpm.UnlockRange(reinterpret_cast<void*>(test_page_size*600+1), test_page_size*500); - lpm.UnlockRange(reinterpret_cast<void*>(test_page_size*1200), test_page_size*500-1); +private: + int count; + int lockedcount; +}; - /* Check that everything is released */ - BOOST_CHECK(lpm.GetLockedPageCount() == 0); +BOOST_AUTO_TEST_CASE(lockedpool_tests_mock) +{ + // Test over three virtual arenas, of which one will succeed being locked + std::unique_ptr<LockedPageAllocator> x(new TestLockedPageAllocator(3, 1)); + LockedPool pool(std::move(x)); + BOOST_CHECK(pool.stats().total == 0); + BOOST_CHECK(pool.stats().locked == 0); - /* A few and unlocks of size zero (should have no effect) */ - addr = 0; - for(int i=0; i<1000; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), 0); - addr += 1; - } - BOOST_CHECK(lpm.GetLockedPageCount() == 0); - addr = 0; - for(int i=0; i<1000; ++i) + void *a0 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a0); + BOOST_CHECK(pool.stats().locked == LockedPool::ARENA_SIZE); + void *a1 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a1); + void *a2 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a2); + void *a3 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a3); + void *a4 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a4); + void *a5 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a5); + // We've passed a count of three arenas, so this allocation should fail + void *a6 = pool.alloc(16); + BOOST_CHECK(!a6); + + pool.free(a0); + pool.free(a2); + pool.free(a4); + pool.free(a1); + pool.free(a3); + pool.free(a5); + BOOST_CHECK(pool.stats().total == 3*LockedPool::ARENA_SIZE); + BOOST_CHECK(pool.stats().locked == LockedPool::ARENA_SIZE); + BOOST_CHECK(pool.stats().used == 0); +} + +// These tests used the live LockedPoolManager object, this is also used +// by other tests so the conditions are somewhat less controllable and thus the +// tests are somewhat more error-prone. +BOOST_AUTO_TEST_CASE(lockedpool_tests_live) +{ + LockedPoolManager &pool = LockedPoolManager::Instance(); + LockedPool::Stats initial = pool.stats(); + + void *a0 = pool.alloc(16); + BOOST_CHECK(a0); + // Test reading and writing the allocated memory + *((uint32_t*)a0) = 0x1234; + BOOST_CHECK(*((uint32_t*)a0) == 0x1234); + + pool.free(a0); + try { // Test exception on double-free + pool.free(a0); + BOOST_CHECK(0); + } catch(std::runtime_error &) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), 0); - addr += 1; } - BOOST_CHECK(lpm.GetLockedPageCount() == 0); - BOOST_CHECK((last_unlock_len & (test_page_size-1)) == 0); // always unlock entire pages + // If more than one new arena was allocated for the above tests, something is wrong + BOOST_CHECK(pool.stats().total <= (initial.total + LockedPool::ARENA_SIZE)); + // Usage must be back to where it started + BOOST_CHECK(pool.stats().used == initial.used); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bctest.py b/src/test/bctest.py index d801415c70..00d96eff19 100644 --- a/src/test/bctest.py +++ b/src/test/bctest.py @@ -6,6 +6,17 @@ import subprocess import os import json import sys +import binascii +import difflib +import logging + +def parse_output(a, fmt): + if fmt == 'json': # json: compare parsed data + return json.loads(a) + elif fmt == 'hex': # hex: parse and compare binary data + return binascii.a2b_hex(a.strip()) + else: + raise NotImplementedError("Don't know how to compare %s" % fmt) def bctest(testDir, testObj, exeext): @@ -23,39 +34,71 @@ def bctest(testDir, testObj, exeext): outputData = None if "output_cmp" in testObj: outputFn = testObj['output_cmp'] - outputData = open(testDir + "/" + outputFn).read() + outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) + try: + outputData = open(testDir + "/" + outputFn).read() + except: + logging.error("Output file " + outputFn + " can not be opened") + raise if not outputData: - print("Output data missing for " + outputFn) - sys.exit(1) + logging.error("Output data missing for " + outputFn) + raise Exception + proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) try: outs = proc.communicate(input=inputData) except OSError: - print("OSError, Failed to execute " + execprog) - sys.exit(1) + logging.error("OSError, Failed to execute " + execprog) + raise - if outputData and (outs[0] != outputData): - print("Output data mismatch for " + outputFn) - sys.exit(1) + if outputData: + try: + a_parsed = parse_output(outs[0], outputType) + except Exception as e: + logging.error('Error parsing command output as %s: %s' % (outputType,e)) + raise + try: + b_parsed = parse_output(outputData, outputType) + except Exception as e: + logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e)) + raise + if a_parsed != b_parsed: + logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") + raise Exception + if outs[0] != outputData: + error_message = "Output formatting mismatch for " + outputFn + ":\n" + error_message += "".join(difflib.context_diff(outputData.splitlines(True), + outs[0].splitlines(True), + fromfile=outputFn, + tofile="returned")) + logging.error(error_message) + raise Exception wantRC = 0 if "return_code" in testObj: wantRC = testObj['return_code'] if proc.returncode != wantRC: - print("Return code mismatch for " + outputFn) - sys.exit(1) + logging.error("Return code mismatch for " + outputFn) + raise Exception -def bctester(testDir, input_basename, buildenv, verbose = False): +def bctester(testDir, input_basename, buildenv): input_filename = testDir + "/" + input_basename raw_data = open(input_filename).read() input_data = json.loads(raw_data) + failed_testcases = [] + for testObj in input_data: - if verbose and "description" in testObj: - print ("Testing: " + testObj["description"]) - bctest(testDir, testObj, buildenv.exeext) - if verbose and "description" in testObj: - print ("PASS") + try: + bctest(testDir, testObj, buildenv.exeext) + logging.info("PASSED: " + testObj["description"]) + except: + logging.info("FAILED: " + testObj["description"]) + failed_testcases.append(testObj["description"]) - sys.exit(0) + if failed_testcases: + logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]") + sys.exit(1) + else: + sys.exit(0) diff --git a/src/test/bitcoin-util-test.py b/src/test/bitcoin-util-test.py index 3099506d6d..9afe91aca0 100755 --- a/src/test/bitcoin-util-test.py +++ b/src/test/bitcoin-util-test.py @@ -4,9 +4,11 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. from __future__ import division,print_function,unicode_literals import os +import sys import bctest import buildenv import argparse +import logging help_text="""Test framework for bitcoin utils. @@ -19,9 +21,9 @@ test/bitcoin-util-test.py --src=[srcdir] if __name__ == '__main__': - verbose = False try: srcdir = os.environ["srcdir"] + verbose = False except: parser = argparse.ArgumentParser(description=help_text) parser.add_argument('-s', '--srcdir') @@ -29,4 +31,13 @@ if __name__ == '__main__': args = parser.parse_args() srcdir = args.srcdir verbose = args.verbose - bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv, verbose = verbose) + + if verbose: + level = logging.DEBUG + else: + level = logging.ERROR + formatter = '%(asctime)s - %(levelname)s - %(message)s' + # Add the format/level to the logger + logging.basicConfig(format = formatter, level=level) + + bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv) diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 7530b013bd..b0d9184816 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -80,8 +80,8 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); - std::list<CTransaction> removed; - pool.removeRecursive(block.vtx[2], removed); + std::vector<std::shared_ptr<const CTransaction>> removed; + pool.removeRecursive(block.vtx[2], &removed); BOOST_CHECK_EQUAL(removed.size(), 1); CBlock block2; diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 033a50f94f..a73dbe725c 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -55,15 +55,15 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) CTxMemPool testPool(CFeeRate(0)); - std::list<CTransaction> removed; + std::vector<std::shared_ptr<const CTransaction>> removed; // Nothing in pool, remove should do nothing: - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 0); // Just the parent: testPool.addUnchecked(txParent.GetHash(), entry.FromTx(txParent)); - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 1); removed.clear(); @@ -75,16 +75,16 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) testPool.addUnchecked(txGrandChild[i].GetHash(), entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: - testPool.removeRecursive(txChild[0], removed); + testPool.removeRecursive(txChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 2); removed.clear(); // ... make sure grandchild and child are gone: - testPool.removeRecursive(txGrandChild[0], removed); + testPool.removeRecursive(txGrandChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 0); - testPool.removeRecursive(txChild[0], removed); + testPool.removeRecursive(txChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 0); // Remove parent, all children/grandchildren should go: - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 5); BOOST_CHECK_EQUAL(testPool.size(), 0); removed.clear(); @@ -97,7 +97,7 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) } // Now remove the parent, as might happen if a block-re-org occurs but the parent cannot be // put into the mempool (maybe because it is non-standard): - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 6); BOOST_CHECK_EQUAL(testPool.size(), 0); removed.clear(); @@ -281,12 +281,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 10); // Now try removing tx10 and verify the sort order returns to normal - std::list<CTransaction> removed; - pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx(), removed); + pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx()); CheckSort<descendant_score>(pool, snapshotOrder); - pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx(), removed); - pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx(), removed); + pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx()); + pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx()); /* Now check the sort on the mining score index. * Final order should be: * @@ -413,8 +412,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) /* after tx6 is mined, tx7 should move up in the sort */ std::vector<CTransaction> vtx; vtx.push_back(tx6); - std::list<CTransaction> dummy; - pool.removeForBlock(vtx, 1, dummy, false); + pool.removeForBlock(vtx, 1, NULL, false); sortedOrder.erase(sortedOrder.begin()+1); sortedOrder.pop_back(); @@ -549,12 +547,11 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7, &pool)); std::vector<CTransaction> vtx; - std::list<CTransaction> conflicts; SetMockTime(42); SetMockTime(42 + CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); // ... we should keep the same min fee until we get a block - pool.removeForBlock(vtx, 1, conflicts); + pool.removeForBlock(vtx, 1); SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/2); // ... then feerate should drop 1/2 each halflife diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index d6d7b5716e..a94979fd77 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -137,8 +137,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, // Test that packages above the min relay fee do get included, even if one // of the transactions is below the min relay fee // Remove the low fee transaction and replace with a higher fee transaction - std::list<CTransaction> dummy; - mempool.removeRecursive(tx, dummy); + mempool.removeRecursive(tx); tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee hashLowFeeTx = tx.GetHash(); mempool.addUnchecked(hashLowFeeTx, entry.Fee(feeToUse+2).FromTx(tx)); diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 5c902387f1..f57c24270c 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -46,7 +46,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (unsigned int i = 0; i < 128; i++) garbage.push_back('X'); CMutableTransaction tx; - std::list<CTransaction> dummyConflicted; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); @@ -81,7 +80,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[9-h].pop_back(); } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); block.clear(); if (blocknum == 30) { // At this point we should need to combine 5 buckets to get enough data points @@ -125,7 +124,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Mine 50 more blocks with no transactions happening, estimates shouldn't change // We haven't decayed the moving average enough so we still have enough data points in every bucket while (blocknum < 250) - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); for (int i = 1; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); @@ -146,7 +145,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].push_back(hash); } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); } int answerFound; @@ -167,7 +166,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265, dummyConflicted); + mpool.removeForBlock(block, 265); block.clear(); for (int i = 1; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); @@ -187,7 +186,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) block.push_back(*ptx); } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); block.clear(); } for (int i = 1; i < 10; i++) { diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 0f1c166abc..313d33507f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -503,7 +503,7 @@ void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants } } -void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list<CTransaction>& removed) +void CTxMemPool::removeRecursive(const CTransaction &origTx, std::vector<std::shared_ptr<const CTransaction>>* removed) { // Remove transaction from memory pool { @@ -530,8 +530,10 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list<CTransact BOOST_FOREACH(txiter it, txToRemove) { CalculateDescendants(it, setAllRemoves); } - BOOST_FOREACH(txiter it, setAllRemoves) { - removed.push_back(it->GetTx()); + if (removed) { + BOOST_FOREACH(txiter it, setAllRemoves) { + removed->emplace_back(it->GetSharedTx()); + } } RemoveStaged(setAllRemoves, false); } @@ -541,7 +543,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem { // Remove transactions spending a coinbase which are now immature and no-longer-final transactions LOCK(cs); - list<CTransaction> transactionsToRemove; + setEntries txToRemove; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { const CTransaction& tx = it->GetTx(); LockPoints lp = it->GetLockPoints(); @@ -549,16 +551,16 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem if (!CheckFinalTx(tx, flags) || !CheckSequenceLocks(tx, flags, &lp, validLP)) { // Note if CheckSequenceLocks fails the LockPoints may still be invalid // So it's critical that we remove the tx and not depend on the LockPoints. - transactionsToRemove.push_back(tx); + txToRemove.insert(it); } else if (it->GetSpendsCoinbase()) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); + if (nCheckFrequency != 0) assert(coins); if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < COINBASE_MATURITY)) { - transactionsToRemove.push_back(tx); + txToRemove.insert(it); break; } } @@ -567,13 +569,14 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem mapTx.modify(it, update_lock_points(lp)); } } - BOOST_FOREACH(const CTransaction& tx, transactionsToRemove) { - list<CTransaction> removed; - removeRecursive(tx, removed); + setEntries setAllRemoves; + for (txiter it : txToRemove) { + CalculateDescendants(it, setAllRemoves); } + RemoveStaged(setAllRemoves, false); } -void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed) +void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<std::shared_ptr<const CTransaction>>* removed) { // Remove transactions which depend on inputs of tx, recursively LOCK(cs); @@ -594,7 +597,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction> * Called when a block is connected. Removes from mempool and updates the miner fee estimator. */ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts, bool fCurrentEstimate) + std::vector<std::shared_ptr<const CTransaction>>* conflicts, bool fCurrentEstimate) { LOCK(cs); std::vector<CTxMemPoolEntry> entries; @@ -830,6 +833,10 @@ void CTxMemPool::queryHashes(vector<uint256>& vtxid) } } +static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator it) { + return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize()), it->GetModifiedFee() - it->GetFee()}; +} + std::vector<TxMempoolInfo> CTxMemPool::infoAll() const { LOCK(cs); @@ -838,7 +845,7 @@ std::vector<TxMempoolInfo> CTxMemPool::infoAll() const std::vector<TxMempoolInfo> ret; ret.reserve(mapTx.size()); for (auto it : iters) { - ret.push_back(TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize())}); + ret.push_back(GetInfo(it)); } return ret; @@ -859,7 +866,7 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const indexed_transaction_set::const_iterator i = mapTx.find(hash); if (i == mapTx.end()) return TxMempoolInfo(); - return TxMempoolInfo{i->GetSharedTx(), i->GetTime(), CFeeRate(i->GetFee(), i->GetTxSize())}; + return GetInfo(i); } CFeeRate CTxMemPool::estimateFee(int nBlocks) const diff --git a/src/txmempool.h b/src/txmempool.h index 1763930ba0..9b0ca4655e 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -6,9 +6,12 @@ #ifndef BITCOIN_TXMEMPOOL_H #define BITCOIN_TXMEMPOOL_H -#include <list> #include <memory> #include <set> +#include <map> +#include <vector> +#include <utility> +#include <string> #include "amount.h" #include "coins.h" @@ -326,6 +329,9 @@ struct TxMempoolInfo /** Feerate of the transaction. */ CFeeRate feeRate; + + /** The fee delta. */ + int64_t nFeeDelta; }; /** @@ -521,11 +527,11 @@ public: bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); - void removeRecursive(const CTransaction &tx, std::list<CTransaction>& removed); + void removeRecursive(const CTransaction &tx, std::vector<std::shared_ptr<const CTransaction>>* removed = NULL); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); - void removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed); + void removeConflicts(const CTransaction &tx, std::vector<std::shared_ptr<const CTransaction>>* removed = NULL); void removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts, bool fCurrentEstimate = true); + std::vector<std::shared_ptr<const CTransaction>>* conflicts = NULL, bool fCurrentEstimate = true); void clear(); void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); diff --git a/src/coincontrol.h b/src/wallet/coincontrol.h index e33adc4d2b..08d23688ff 100644 --- a/src/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_COINCONTROL_H -#define BITCOIN_COINCONTROL_H +#ifndef BITCOIN_WALLET_COINCONTROL_H +#define BITCOIN_WALLET_COINCONTROL_H #include "primitives/transaction.h" @@ -22,6 +22,8 @@ public: bool fOverrideFeeRate; //! Feerate to use if overrideFeeRate is true CFeeRate nFeeRate; + //! Override the default confirmation target, 0 = use default + int nConfirmTarget; CCoinControl() { @@ -37,6 +39,7 @@ public: nMinimumTotalFee = 0; nFeeRate = CFeeRate(0); fOverrideFeeRate = false; + nConfirmTarget = 0; } bool HasSelected() const @@ -73,4 +76,4 @@ private: std::set<COutPoint> setSelected; }; -#endif // BITCOIN_COINCONTROL_H +#endif // BITCOIN_WALLET_COINCONTROL_H diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 190f8ecf2a..31ee060677 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -48,12 +48,12 @@ bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::v int i = 0; if (nDerivationMethod == 0) - i = BytesToKeySHA512AES(chSalt, strKeyData, nRounds, chKey, chIV); + i = BytesToKeySHA512AES(chSalt, strKeyData, nRounds, vchKey.data(), vchIV.data()); if (i != (int)WALLET_CRYPTO_KEY_SIZE) { - memory_cleanse(chKey, sizeof(chKey)); - memory_cleanse(chIV, sizeof(chIV)); + memory_cleanse(vchKey.data(), vchKey.size()); + memory_cleanse(vchIV.data(), vchIV.size()); return false; } @@ -66,8 +66,8 @@ bool CCrypter::SetKey(const CKeyingMaterial& chNewKey, const std::vector<unsigne if (chNewKey.size() != WALLET_CRYPTO_KEY_SIZE || chNewIV.size() != WALLET_CRYPTO_IV_SIZE) return false; - memcpy(&chKey[0], &chNewKey[0], sizeof chKey); - memcpy(&chIV[0], &chNewIV[0], sizeof chIV); + memcpy(vchKey.data(), chNewKey.data(), chNewKey.size()); + memcpy(vchIV.data(), chNewIV.data(), chNewIV.size()); fKeySet = true; return true; @@ -82,7 +82,7 @@ bool CCrypter::Encrypt(const CKeyingMaterial& vchPlaintext, std::vector<unsigned // n + AES_BLOCKSIZE bytes vchCiphertext.resize(vchPlaintext.size() + AES_BLOCKSIZE); - AES256CBCEncrypt enc(chKey, chIV, true); + AES256CBCEncrypt enc(vchKey.data(), vchIV.data(), true); size_t nLen = enc.Encrypt(&vchPlaintext[0], vchPlaintext.size(), &vchCiphertext[0]); if(nLen < vchPlaintext.size()) return false; @@ -101,7 +101,7 @@ bool CCrypter::Decrypt(const std::vector<unsigned char>& vchCiphertext, CKeyingM vchPlaintext.resize(nLen); - AES256CBCDecrypt dec(chKey, chIV, true); + AES256CBCDecrypt dec(vchKey.data(), vchIV.data(), true); nLen = dec.Decrypt(&vchCiphertext[0], vchCiphertext.size(), &vchPlaintext[0]); if(nLen == 0) return false; diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 5d0a4a3305..f00f7fa731 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -77,8 +77,8 @@ class CCrypter { friend class wallet_crypto::TestCrypter; // for test access to chKey/chIV private: - unsigned char chKey[WALLET_CRYPTO_KEY_SIZE]; - unsigned char chIV[WALLET_CRYPTO_IV_SIZE]; + std::vector<unsigned char, secure_allocator<unsigned char>> vchKey; + std::vector<unsigned char, secure_allocator<unsigned char>> vchIV; bool fKeySet; int BytesToKeySHA512AES(const std::vector<unsigned char>& chSalt, const SecureString& strKeyData, int count, unsigned char *key,unsigned char *iv) const; @@ -91,28 +91,21 @@ public: void CleanKey() { - memory_cleanse(chKey, sizeof(chKey)); - memory_cleanse(chIV, sizeof(chIV)); + memory_cleanse(vchKey.data(), vchKey.size()); + memory_cleanse(vchIV.data(), vchIV.size()); fKeySet = false; } CCrypter() { fKeySet = false; - - // Try to keep the key data out of swap (and be a bit over-careful to keep the IV that we don't even use out of swap) - // Note that this does nothing about suspend-to-disk (which will put all our key data on disk) - // Note as well that at no point in this program is any attempt made to prevent stealing of keys by reading the memory of the running process. - LockedPageManager::Instance().LockRange(&chKey[0], sizeof chKey); - LockedPageManager::Instance().LockRange(&chIV[0], sizeof chIV); + vchKey.resize(WALLET_CRYPTO_KEY_SIZE); + vchIV.resize(WALLET_CRYPTO_IV_SIZE); } ~CCrypter() { CleanKey(); - - LockedPageManager::Instance().UnlockRange(&chKey[0], sizeof chKey); - LockedPageManager::Instance().UnlockRange(&chIV[0], sizeof chIV); } }; diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 8a1bbd5684..b638810e9d 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -24,6 +24,7 @@ #include <univalue.h> +#include <boost/assign/list_of.hpp> #include <boost/foreach.hpp> using namespace std; @@ -637,3 +638,424 @@ UniValue dumpwallet(const JSONRPCRequest& request) file.close(); return NullUniValue; } + + +UniValue processImport(const UniValue& data) { + try { + bool success = false; + + // Required fields. + const UniValue& scriptPubKey = data["scriptPubKey"]; + + // Should have script or JSON with "address". + if (!(scriptPubKey.getType() == UniValue::VOBJ && scriptPubKey.exists("address")) && !(scriptPubKey.getType() == UniValue::VSTR)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid scriptPubKey"); + } + + // Optional fields. + const string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; + const UniValue& pubKeys = data.exists("pubkeys") ? data["pubkeys"].get_array() : UniValue(); + const UniValue& keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); + const bool& internal = data.exists("internal") ? data["internal"].get_bool() : false; + const bool& watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; + const string& label = data.exists("label") && !internal ? data["label"].get_str() : ""; + const int64_t& timestamp = data.exists("timestamp") && data["timestamp"].get_int64() > 1 ? data["timestamp"].get_int64() : 1; + + bool isScript = scriptPubKey.getType() == UniValue::VSTR; + bool isP2SH = strRedeemScript.length() > 0; + const string& output = isScript ? scriptPubKey.get_str() : scriptPubKey["address"].get_str(); + + // Parse the output. + CScript script; + CBitcoinAddress address; + + if (!isScript) { + address = CBitcoinAddress(output); + script = GetScriptForDestination(address.Get()); + } else { + if (!IsHex(output)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid scriptPubKey"); + } + + std::vector<unsigned char> vData(ParseHex(output)); + script = CScript(vData.begin(), vData.end()); + } + + // Watchonly and private keys + if (watchOnly && keys.size()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Incompatibility found between watchonly and keys"); + } + + // Internal + Label + if (internal && data.exists("label")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Incompatibility found between internal and label"); + } + + // Not having Internal + Script + if (!internal && isScript) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal must be set for hex scriptPubKey"); + } + + // Keys / PubKeys size check. + if (!isP2SH && (keys.size() > 1 || pubKeys.size() > 1)) { // Address / scriptPubKey + throw JSONRPCError(RPC_INVALID_PARAMETER, "More than private key given for one address"); + } + + // Invalid P2SH redeemScript + if (isP2SH && !IsHex(strRedeemScript)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid redeem script"); + } + + // Process. // + + // P2SH + if (isP2SH) { + // Import redeem script. + std::vector<unsigned char> vData(ParseHex(strRedeemScript)); + CScript redeemScript = CScript(vData.begin(), vData.end()); + + // Invalid P2SH address + if (!script.IsPayToScriptHash()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid P2SH address / script"); + } + + pwalletMain->MarkDirty(); + + if (!pwalletMain->HaveWatchOnly(redeemScript) && !pwalletMain->AddWatchOnly(redeemScript)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } + + if (!pwalletMain->HaveCScript(redeemScript) && !pwalletMain->AddCScript(redeemScript)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding p2sh redeemScript to wallet"); + } + + CBitcoinAddress redeemAddress = CBitcoinAddress(CScriptID(redeemScript)); + CScript redeemDestination = GetScriptForDestination(redeemAddress.Get()); + + if (::IsMine(*pwalletMain, redeemDestination) == ISMINE_SPENDABLE) { + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); + } + + pwalletMain->MarkDirty(); + + if (!pwalletMain->HaveWatchOnly(redeemDestination) && !pwalletMain->AddWatchOnly(redeemDestination)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } + + // add to address book or update label + if (address.IsValid()) { + pwalletMain->SetAddressBook(address.Get(), label, "receive"); + } + + // Import private keys. + if (keys.size()) { + for (size_t i = 0; i < keys.size(); i++) { + const string& privkey = keys[i].get_str(); + + CBitcoinSecret vchSecret; + bool fGood = vchSecret.SetString(privkey); + + if (!fGood) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); + } + + CKey key = vchSecret.GetKey(); + + if (!key.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Private key outside allowed range"); + } + + CPubKey pubkey = key.GetPubKey(); + assert(key.VerifyPubKey(pubkey)); + + CKeyID vchAddress = pubkey.GetID(); + pwalletMain->MarkDirty(); + pwalletMain->SetAddressBook(vchAddress, label, "receive"); + + if (pwalletMain->HaveKey(vchAddress)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Already have this key"); + } + + pwalletMain->mapKeyMetadata[vchAddress].nCreateTime = timestamp; + + if (!pwalletMain->AddKeyPubKey(key, pubkey)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); + } + + if (timestamp < pwalletMain->nTimeFirstKey) { + pwalletMain->nTimeFirstKey = timestamp; + } + } + } + + success = true; + } else { + // Import public keys. + if (pubKeys.size() && keys.size() == 0) { + const string& strPubKey = pubKeys[0].get_str(); + + if (!IsHex(strPubKey)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey must be a hex string"); + } + + std::vector<unsigned char> data(ParseHex(strPubKey)); + CPubKey pubKey(data.begin(), data.end()); + + if (!pubKey.IsFullyValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey is not a valid public key"); + } + + CBitcoinAddress pubKeyAddress = CBitcoinAddress(pubKey.GetID()); + + // Consistency check. + if (!isScript && !(pubKeyAddress.Get() == address.Get())) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Consistency check failed"); + } + + // Consistency check. + if (isScript) { + CBitcoinAddress scriptAddress; + CTxDestination destination; + + if (ExtractDestination(script, destination)) { + scriptAddress = CBitcoinAddress(destination); + if (!(scriptAddress.Get() == pubKeyAddress.Get())) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Consistency check failed"); + } + } + } + + CScript pubKeyScript = GetScriptForDestination(pubKeyAddress.Get()); + + if (::IsMine(*pwalletMain, pubKeyScript) == ISMINE_SPENDABLE) { + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); + } + + pwalletMain->MarkDirty(); + + if (!pwalletMain->HaveWatchOnly(pubKeyScript) && !pwalletMain->AddWatchOnly(pubKeyScript)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } + + // add to address book or update label + if (pubKeyAddress.IsValid()) { + pwalletMain->SetAddressBook(pubKeyAddress.Get(), label, "receive"); + } + + // TODO Is this necessary? + CScript scriptRawPubKey = GetScriptForRawPubKey(pubKey); + + if (::IsMine(*pwalletMain, scriptRawPubKey) == ISMINE_SPENDABLE) { + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); + } + + pwalletMain->MarkDirty(); + + if (!pwalletMain->HaveWatchOnly(scriptRawPubKey) && !pwalletMain->AddWatchOnly(scriptRawPubKey)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } + + success = true; + } + + // Import private keys. + if (keys.size()) { + const string& strPrivkey = keys[0].get_str(); + + // Checks. + CBitcoinSecret vchSecret; + bool fGood = vchSecret.SetString(strPrivkey); + + if (!fGood) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); + } + + CKey key = vchSecret.GetKey(); + if (!key.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Private key outside allowed range"); + } + + CPubKey pubKey = key.GetPubKey(); + assert(key.VerifyPubKey(pubKey)); + + CBitcoinAddress pubKeyAddress = CBitcoinAddress(pubKey.GetID()); + + // Consistency check. + if (!isScript && !(pubKeyAddress.Get() == address.Get())) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Consistency check failed"); + } + + // Consistency check. + if (isScript) { + CBitcoinAddress scriptAddress; + CTxDestination destination; + + if (ExtractDestination(script, destination)) { + scriptAddress = CBitcoinAddress(destination); + if (!(scriptAddress.Get() == pubKeyAddress.Get())) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Consistency check failed"); + } + } + } + + CKeyID vchAddress = pubKey.GetID(); + pwalletMain->MarkDirty(); + pwalletMain->SetAddressBook(vchAddress, label, "receive"); + + if (pwalletMain->HaveKey(vchAddress)) { + return false; + } + + pwalletMain->mapKeyMetadata[vchAddress].nCreateTime = timestamp; + + if (!pwalletMain->AddKeyPubKey(key, pubKey)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); + } + + if (timestamp < pwalletMain->nTimeFirstKey) { + pwalletMain->nTimeFirstKey = timestamp; + } + + success = true; + } + + // Import scriptPubKey only. + if (pubKeys.size() == 0 && keys.size() == 0) { + if (::IsMine(*pwalletMain, script) == ISMINE_SPENDABLE) { + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); + } + + pwalletMain->MarkDirty(); + + if (!pwalletMain->HaveWatchOnly(script) && !pwalletMain->AddWatchOnly(script)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } + + if (scriptPubKey.getType() == UniValue::VOBJ) { + // add to address book or update label + if (address.IsValid()) { + pwalletMain->SetAddressBook(address.Get(), label, "receive"); + } + } + + success = true; + } + } + + UniValue result = UniValue(UniValue::VOBJ); + result.pushKV("success", UniValue(success)); + return result; + } catch (const UniValue& e) { + UniValue result = UniValue(UniValue::VOBJ); + result.pushKV("success", UniValue(false)); + result.pushKV("error", e); + return result; + } catch (...) { + UniValue result = UniValue(UniValue::VOBJ); + result.pushKV("success", UniValue(false)); + result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, "Missing required fields")); + return result; + } +} + +UniValue importmulti(const JSONRPCRequest& mainRequest) +{ + // clang-format off + if (mainRequest.fHelp || mainRequest.params.size() < 1 || mainRequest.params.size() > 2) + throw runtime_error( + "importmulti '[<json import requests>]' '<json options>' \n\n" + "Import addresses/scripts (with private or public keys, redeem script (P2SH)), rescanning all addresses in one-shot-only (rescan can be disabled via options).\n\n" + "Arguments:\n" + "1. request array (array, required) Data to be imported\n" + " [ (array of json objects)\n" + " {\n" + " \"scriptPubKey\": \"<script>\" | { \"address\":\"<address>\" }, (string / json, required) Type of scriptPubKey (string for script, json for address)\n" + " \"redeemscript\": \"<script>\" , (string, optional) Allowed only if the scriptPubKey is a P2SH address or a P2SH scriptPubKey\n" + " \"pubkeys\": [\"<pubKey>\", ... ] , (array, optional) Array of strings giving pubkeys that must occur in the output or redeemscript\n" + " \"keys\": [\"<key>\", ... ] , (array, optional) Array of strings giving private keys whose corresponding public keys must occur in the output or redeemscript\n" + " \"internal\": <true> , (boolean, optional, default: false) Stating whether matching outputs should be be treated as not incoming payments\n" + " \"watchonly\": <true> , (boolean, optional, default: false) Stating whether matching outputs should be considered watched even when they're not spendable, only allowed if keys are empty\n" + " \"label\": <label> , (string, optional, default: '') Label to assign to the address (aka account name, for now), only allowed with internal=false\n" + " \"timestamp\": 1454686740, (integer, optional, default now) Timestamp\n" + " }\n" + " ,...\n" + " ]\n" + "2. json options (json, optional)\n" + " {\n" + " \"rescan\": <false>, (boolean, optional, default: true) Stating if should rescan the blockchain after all imports\n" + " }\n" + "\nExamples:\n" + + HelpExampleCli("importmulti", "'[{ \"scriptPubKey\": { \"address\": \"<my address>\" }, \"timestamp\":1455191478 }, " + "{ \"scriptPubKey\": { \"address\": \"<my 2nd address>\" }, \"label\": \"example 2\", \"timestamp\": 1455191480 }]'") + + HelpExampleCli("importmulti", "'[{ \"scriptPubKey\": { \"address\": \"<my address>\" }, \"timestamp\":1455191478 }]' '{ \"rescan\": false}'") + + + "\nResponse is an array with the same size as the input that has the execution result :\n" + " [{ \"success\": true } , { \"success\": false, \"error\": { \"code\": -1, \"message\": \"Internal Server Error\"} }, ... ]\n"); + + // clang-format on + if (!EnsureWalletIsAvailable(mainRequest.fHelp)) { + return NullUniValue; + } + + RPCTypeCheck(mainRequest.params, boost::assign::list_of(UniValue::VARR)(UniValue::VOBJ)); + + const UniValue& requests = mainRequest.params[0]; + + //Default options + bool fRescan = true; + + if (mainRequest.params.size() > 1) { + const UniValue& options = mainRequest.params[1]; + + if (options.exists("rescan")) { + fRescan = options["rescan"].get_bool(); + } + } + + LOCK2(cs_main, pwalletMain->cs_wallet); + EnsureWalletIsUnlocked(); + + bool fRunScan = false; + const int64_t minimumTimestamp = 1; + int64_t nLowestTimestamp; + + if (fRescan && chainActive.Tip()) { + nLowestTimestamp = chainActive.Tip()->GetBlockTime(); + } else { + fRescan = false; + } + + UniValue response(UniValue::VARR); + + BOOST_FOREACH (const UniValue& data, requests.getValues()) { + const UniValue result = processImport(data); + response.push_back(result); + + if (!fRescan) { + continue; + } + + // If at least one request was successful then allow rescan. + if (result["success"].get_bool()) { + fRunScan = true; + } + + // Get the lowest timestamp. + const int64_t& timestamp = data.exists("timestamp") && data["timestamp"].get_int64() > minimumTimestamp ? data["timestamp"].get_int64() : minimumTimestamp; + + if (timestamp < nLowestTimestamp) { + nLowestTimestamp = timestamp; + } + } + + if (fRescan && fRunScan && requests.size() && nLowestTimestamp <= chainActive.Tip()->GetBlockTime()) { + CBlockIndex* pindex = nLowestTimestamp > minimumTimestamp ? chainActive.FindLatestBefore(nLowestTimestamp) : chainActive.Genesis(); + + if (pindex) { + pwalletMain->ScanForWalletTransactions(pindex, true); + pwalletMain->ReacceptWalletTransactions(); + } + } + + return response; +} diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 33620aa6ff..5a22e0278d 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -6,6 +6,7 @@ #include "amount.h" #include "base58.h" #include "chain.h" +#include "consensus/validation.h" #include "core_io.h" #include "init.h" #include "main.h" @@ -362,11 +363,14 @@ static void SendMoney(const CTxDestination &address, CAmount nValue, bool fSubtr vecSend.push_back(recipient); if (!pwalletMain->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError)) { if (!fSubtractFeeFromAmount && nValue + nFeeRequired > pwalletMain->GetBalance()) - strError = strprintf("Error: This transaction requires a transaction fee of at least %s because of its amount, complexity, or use of recently received funds!", FormatMoney(nFeeRequired)); + strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); + throw JSONRPCError(RPC_WALLET_ERROR, strError); + } + CValidationState state; + if (!pwalletMain->CommitTransaction(wtxNew, reservekey, g_connman.get(), state)) { + strError = strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason()); throw JSONRPCError(RPC_WALLET_ERROR, strError); } - if (!pwalletMain->CommitTransaction(wtxNew, reservekey, g_connman.get())) - throw JSONRPCError(RPC_WALLET_ERROR, "Error: The transaction was rejected! This might happen if some of the coins in your wallet were already spent, such as if you used a copy of the wallet and coins were spent in the copy but not marked as spent here."); } UniValue sendtoaddress(const JSONRPCRequest& request) @@ -959,8 +963,11 @@ UniValue sendmany(const JSONRPCRequest& request) bool fCreated = pwalletMain->CreateTransaction(vecSend, wtx, keyChange, nFeeRequired, nChangePosRet, strFailReason); if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); - if (!pwalletMain->CommitTransaction(wtx, keyChange, g_connman.get())) - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction commit failed"); + CValidationState state; + if (!pwalletMain->CommitTransaction(wtx, keyChange, g_connman.get(), state)) { + strFailReason = strprintf("Transaction commit failed:: %s", state.GetRejectReason()); + throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); + } return wtx.GetHash().GetHex(); } @@ -2279,7 +2286,7 @@ UniValue getwalletinfo(const JSONRPCRequest& request) " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since GMT epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" @@ -2583,6 +2590,7 @@ extern UniValue dumpwallet(const JSONRPCRequest& request); extern UniValue importwallet(const JSONRPCRequest& request); extern UniValue importprunedfunds(const JSONRPCRequest& request); extern UniValue removeprunedfunds(const JSONRPCRequest& request); +extern UniValue importmulti(const JSONRPCRequest& request); static const CRPCCommand commands[] = { // category name actor (function) okSafeMode @@ -2607,6 +2615,7 @@ static const CRPCCommand commands[] = { "wallet", "gettransaction", &gettransaction, false }, { "wallet", "getunconfirmedbalance", &getunconfirmedbalance, false }, { "wallet", "getwalletinfo", &getwalletinfo, false }, + { "wallet", "importmulti", &importmulti, true }, { "wallet", "importprivkey", &importprivkey, true }, { "wallet", "importwallet", &importwallet, true }, { "wallet", "importaddress", &importaddress, true }, diff --git a/src/wallet/test/crypto_tests.cpp b/src/wallet/test/crypto_tests.cpp index c5f55ef5f0..ce35c53c48 100644 --- a/src/wallet/test/crypto_tests.cpp +++ b/src/wallet/test/crypto_tests.cpp @@ -97,10 +97,10 @@ static void TestPassphraseSingle(const std::vector<unsigned char>& vchSalt, cons OldSetKeyFromPassphrase(passphrase, vchSalt, rounds, 0, chKey, chIV); - BOOST_CHECK_MESSAGE(memcmp(chKey, crypt.chKey, sizeof(chKey)) == 0, \ - HexStr(chKey, chKey+sizeof(chKey)) + std::string(" != ") + HexStr(crypt.chKey, crypt.chKey + (sizeof crypt.chKey))); - BOOST_CHECK_MESSAGE(memcmp(chIV, crypt.chIV, sizeof(chIV)) == 0, \ - HexStr(chIV, chIV+sizeof(chIV)) + std::string(" != ") + HexStr(crypt.chIV, crypt.chIV + (sizeof crypt.chIV))); + BOOST_CHECK_MESSAGE(memcmp(chKey, crypt.vchKey.data(), crypt.vchKey.size()) == 0, \ + HexStr(chKey, chKey+sizeof(chKey)) + std::string(" != ") + HexStr(crypt.vchKey)); + BOOST_CHECK_MESSAGE(memcmp(chIV, crypt.vchIV.data(), crypt.vchIV.size()) == 0, \ + HexStr(chIV, chIV+sizeof(chIV)) + std::string(" != ") + HexStr(crypt.vchIV)); if(!correctKey.empty()) BOOST_CHECK_MESSAGE(memcmp(chKey, &correctKey[0], sizeof(chKey)) == 0, \ @@ -127,7 +127,7 @@ static void TestDecrypt(const CCrypter& crypt, const std::vector<unsigned char>& CKeyingMaterial vchDecrypted2; int result1, result2; result1 = crypt.Decrypt(vchCiphertext, vchDecrypted1); - result2 = OldDecrypt(vchCiphertext, vchDecrypted2, crypt.chKey, crypt.chIV); + result2 = OldDecrypt(vchCiphertext, vchDecrypted2, crypt.vchKey.data(), crypt.vchIV.data()); BOOST_CHECK(result1 == result2); // These two should be equal. However, OpenSSL 1.0.1j introduced a change @@ -152,7 +152,7 @@ static void TestEncryptSingle(const CCrypter& crypt, const CKeyingMaterial& vchP std::vector<unsigned char> vchCiphertext2; int result1 = crypt.Encrypt(vchPlaintext, vchCiphertext1); - int result2 = OldEncrypt(vchPlaintext, vchCiphertext2, crypt.chKey, crypt.chIV); + int result2 = OldEncrypt(vchPlaintext, vchCiphertext2, crypt.vchKey.data(), crypt.vchIV.data()); BOOST_CHECK(result1 == result2); BOOST_CHECK(vchCiphertext1 == vchCiphertext2); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 66f92d887b..c2bac6e330 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -8,7 +8,7 @@ #include "base58.h" #include "checkpoints.h" #include "chain.h" -#include "coincontrol.h" +#include "wallet/coincontrol.h" #include "consensus/consensus.h" #include "consensus/validation.h" #include "key.h" @@ -658,8 +658,79 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) DBErrors CWallet::ReorderTransactions() { + LOCK(cs_wallet); CWalletDB walletdb(strWalletFile); - return walletdb.ReorderTransactions(this); + + // Old wallets didn't have any defined order for transactions + // Probably a bad idea to change the output of this + + // First: get all CWalletTx and CAccountingEntry into a sorted-by-time multimap. + typedef pair<CWalletTx*, CAccountingEntry*> TxPair; + typedef multimap<int64_t, TxPair > TxItems; + TxItems txByTime; + + for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + { + CWalletTx* wtx = &((*it).second); + txByTime.insert(make_pair(wtx->nTimeReceived, TxPair(wtx, (CAccountingEntry*)0))); + } + list<CAccountingEntry> acentries; + walletdb.ListAccountCreditDebit("", acentries); + BOOST_FOREACH(CAccountingEntry& entry, acentries) + { + txByTime.insert(make_pair(entry.nTime, TxPair((CWalletTx*)0, &entry))); + } + + nOrderPosNext = 0; + std::vector<int64_t> nOrderPosOffsets; + for (TxItems::iterator it = txByTime.begin(); it != txByTime.end(); ++it) + { + CWalletTx *const pwtx = (*it).second.first; + CAccountingEntry *const pacentry = (*it).second.second; + int64_t& nOrderPos = (pwtx != 0) ? pwtx->nOrderPos : pacentry->nOrderPos; + + if (nOrderPos == -1) + { + nOrderPos = nOrderPosNext++; + nOrderPosOffsets.push_back(nOrderPos); + + if (pwtx) + { + if (!walletdb.WriteTx(*pwtx)) + return DB_LOAD_FAIL; + } + else + if (!walletdb.WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) + return DB_LOAD_FAIL; + } + else + { + int64_t nOrderPosOff = 0; + BOOST_FOREACH(const int64_t& nOffsetStart, nOrderPosOffsets) + { + if (nOrderPos >= nOffsetStart) + ++nOrderPosOff; + } + nOrderPos += nOrderPosOff; + nOrderPosNext = std::max(nOrderPosNext, nOrderPos + 1); + + if (!nOrderPosOff) + continue; + + // Since we're changing the order, write it back + if (pwtx) + { + if (!walletdb.WriteTx(*pwtx)) + return DB_LOAD_FAIL; + } + else + if (!walletdb.WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) + return DB_LOAD_FAIL; + } + } + walletdb.WriteOrderPosNext(nOrderPosNext); + + return DB_LOAD_OK; } int64_t CWallet::IncOrderPosNext(CWalletDB *pwalletdb) @@ -1463,7 +1534,8 @@ void CWallet::ReacceptWalletTransactions() CWalletTx& wtx = *(item.second); LOCK(mempool.cs); - wtx.AcceptToMemoryPool(maxTxFee); + CValidationState state; + wtx.AcceptToMemoryPool(maxTxFee, state); } } @@ -2433,17 +2505,22 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt dPriority = wtxNew.ComputePriority(dPriority, nBytes); + // Allow to override the default confirmation target over the CoinControl instance + int currentConfirmationTarget = nTxConfirmTarget; + if (coinControl && coinControl->nConfirmTarget > 0) + currentConfirmationTarget = coinControl->nConfirmTarget; + // Can we complete this as a free transaction? if (fSendFreeTransactions && nBytes <= MAX_FREE_TRANSACTION_CREATE_SIZE) { // Not enough fee: enough priority? - double dPriorityNeeded = mempool.estimateSmartPriority(nTxConfirmTarget); + double dPriorityNeeded = mempool.estimateSmartPriority(currentConfirmationTarget); // Require at least hard-coded AllowFree. if (dPriority >= dPriorityNeeded && AllowFree(dPriority)) break; } - CAmount nFeeNeeded = GetMinimumFee(nBytes, nTxConfirmTarget, mempool); + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, mempool); if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { nFeeNeeded = coinControl->nMinimumTotalFee; } @@ -2474,7 +2551,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt /** * Call after CreateTransaction unless you want to abort */ -bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman) +bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman, CValidationState& state) { { LOCK2(cs_main, cs_wallet); @@ -2502,9 +2579,9 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon if (fBroadcastTransactions) { // Broadcast - if (!wtxNew.AcceptToMemoryPool(maxTxFee)) { + if (!wtxNew.AcceptToMemoryPool(maxTxFee, state)) { // This must not fail. The transaction has already been signed and recorded. - LogPrintf("CommitTransaction(): Error: Transaction not valid\n"); + LogPrintf("CommitTransaction(): Error: Transaction not valid, %s\n", state.GetRejectReason()); return false; } wtxNew.RelayWalletTransaction(connman); @@ -3659,8 +3736,7 @@ int CMerkleTx::GetBlocksToMaturity() const } -bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee) +bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) { - CValidationState state; return ::AcceptToMemoryPool(mempool, state, *this, true, NULL, false, nAbsurdFee); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 5e50eaedc3..d7d1f5513a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -216,7 +216,7 @@ public: bool IsInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet) > 0; } int GetBlocksToMaturity() const; /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(const CAmount& nAbsurdFee); + bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); bool hashUnset() const { return (hashBlock.IsNull() || hashBlock == ABANDON_HASH); } bool isAbandoned() const { return (hashBlock == ABANDON_HASH); } void setAbandoned() { hashBlock = ABANDON_HASH; } @@ -775,7 +775,7 @@ public: */ bool CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletTx& wtxNew, CReserveKey& reservekey, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, const CCoinControl *coinControl = NULL, bool sign = true); - bool CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman); + bool CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman, CValidationState& state); void ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries); bool AddAccountingEntry(const CAccountingEntry&); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 80bfe8255d..43fd6a20ad 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -251,82 +251,6 @@ void CWalletDB::ListAccountCreditDebit(const string& strAccount, list<CAccountin pcursor->close(); } -DBErrors CWalletDB::ReorderTransactions(CWallet* pwallet) -{ - LOCK(pwallet->cs_wallet); - // Old wallets didn't have any defined order for transactions - // Probably a bad idea to change the output of this - - // First: get all CWalletTx and CAccountingEntry into a sorted-by-time multimap. - typedef pair<CWalletTx*, CAccountingEntry*> TxPair; - typedef multimap<int64_t, TxPair > TxItems; - TxItems txByTime; - - for (map<uint256, CWalletTx>::iterator it = pwallet->mapWallet.begin(); it != pwallet->mapWallet.end(); ++it) - { - CWalletTx* wtx = &((*it).second); - txByTime.insert(make_pair(wtx->nTimeReceived, TxPair(wtx, (CAccountingEntry*)0))); - } - list<CAccountingEntry> acentries; - ListAccountCreditDebit("", acentries); - BOOST_FOREACH(CAccountingEntry& entry, acentries) - { - txByTime.insert(make_pair(entry.nTime, TxPair((CWalletTx*)0, &entry))); - } - - int64_t& nOrderPosNext = pwallet->nOrderPosNext; - nOrderPosNext = 0; - std::vector<int64_t> nOrderPosOffsets; - for (TxItems::iterator it = txByTime.begin(); it != txByTime.end(); ++it) - { - CWalletTx *const pwtx = (*it).second.first; - CAccountingEntry *const pacentry = (*it).second.second; - int64_t& nOrderPos = (pwtx != 0) ? pwtx->nOrderPos : pacentry->nOrderPos; - - if (nOrderPos == -1) - { - nOrderPos = nOrderPosNext++; - nOrderPosOffsets.push_back(nOrderPos); - - if (pwtx) - { - if (!WriteTx(*pwtx)) - return DB_LOAD_FAIL; - } - else - if (!WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) - return DB_LOAD_FAIL; - } - else - { - int64_t nOrderPosOff = 0; - BOOST_FOREACH(const int64_t& nOffsetStart, nOrderPosOffsets) - { - if (nOrderPos >= nOffsetStart) - ++nOrderPosOff; - } - nOrderPos += nOrderPosOff; - nOrderPosNext = std::max(nOrderPosNext, nOrderPos + 1); - - if (!nOrderPosOff) - continue; - - // Since we're changing the order, write it back - if (pwtx) - { - if (!WriteTx(*pwtx)) - return DB_LOAD_FAIL; - } - else - if (!WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) - return DB_LOAD_FAIL; - } - } - WriteOrderPosNext(nOrderPosNext); - - return DB_LOAD_OK; -} - class CWalletScanState { public: unsigned int nKeys; @@ -711,7 +635,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) WriteVersion(CLIENT_VERSION); if (wss.fAnyUnordered) - result = ReorderTransactions(pwallet); + result = pwallet->ReorderTransactions(); pwallet->laccentries.clear(); ListAccountCreditDebit("*", pwallet->laccentries); diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 9c9d4922a4..a0525bd9a7 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -153,6 +153,7 @@ public: /// This writes directly to the database, and will not update the CWallet's cached accounting entries! /// Use wallet.AddAccountingEntry instead, to write *and* update its caches. + bool WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry); bool WriteAccountingEntry_Backend(const CAccountingEntry& acentry); bool ReadAccount(const std::string& strAccount, CAccount& account); bool WriteAccount(const std::string& strAccount, const CAccount& account); @@ -165,7 +166,6 @@ public: CAmount GetAccountCreditDebit(const std::string& strAccount); void ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& acentries); - DBErrors ReorderTransactions(CWallet* pwallet); DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(CWallet* pwallet, std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); DBErrors ZapWalletTx(CWallet* pwallet, std::vector<CWalletTx>& vWtx); @@ -180,7 +180,6 @@ private: CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); - bool WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry); }; void ThreadFlushWalletDB(const std::string& strFile); |