diff options
44 files changed, 836 insertions, 426 deletions
diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index e9816f7d19..c664cf81fa 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -175,6 +175,7 @@ def main(): if info is None: exit(1) title = info['title'].strip() + body = info['body'].strip() # precedence order for destination branch argument: # - command line argument # - githubmerge.branch setting @@ -229,6 +230,7 @@ def main(): firstline = 'Merge #%s' % (pull,) message = firstline + '\n\n' message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') + message += '\n\nPull request description:\n\n ' + body.replace('\n', '\n ') + '\n' try: subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch]) except subprocess.CalledProcessError as e: diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh index c2cc2b7013..409f517c9f 100755 --- a/contrib/verifybinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -3,7 +3,8 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -### This script attempts to download the signature file SHA256SUMS.asc from bitcoin.org +### This script attempts to download the signature file SHA256SUMS.asc from +### bitcoincore.org and bitcoin.org and compares them. ### It first checks if the signature passes, and then downloads the files specified in ### the file, and checks if the hashes of these files match those that are specified ### in the signature file. @@ -22,7 +23,9 @@ TMPFILE="hashes.tmp" SIGNATUREFILENAME="SHA256SUMS.asc" RCSUBDIR="test" -BASEDIR="https://bitcoin.org/bin/" +HOST1="https://bitcoincore.org" +HOST2="https://bitcoin.org" +BASEDIR="/bin/" VERSIONPREFIX="bitcoin-core-" RCVERSIONSTRING="rc" @@ -81,7 +84,7 @@ else fi #first we fetch the file containing the signature -WGETOUT=$(wget -N "$BASEDIR$SIGNATUREFILENAME" 2>&1) +WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1) #and then see if wget completed successfully if [ $? -ne 0 ]; then @@ -92,6 +95,22 @@ if [ $? -ne 0 ]; then exit 2 fi +WGETOUT=$(wget -N -O "$SIGNATUREFILENAME.2" "$HOST2$BASEDIR$SIGNATUREFILENAME" 2>&1) +if [ $? -ne 0 ]; then + echo "bitcoin.org failed to provide signature file, but bitcoincore.org did?" + echo "wget output:" + echo "$WGETOUT"|sed 's/^/\t/g' + clean_up $SIGNATUREFILENAME + exit 3 +fi + +SIGFILEDIFFS="$(diff $SIGNATUREFILENAME $SIGNATUREFILENAME.2)" +if [ "$SIGFILEDIFFS" != "" ]; then + echo "bitcoin.org and bitcoincore.org signature files were not equal?" + clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 + exit 4 +fi + #then we check it GPGOUT=$(gpg --yes --decrypt --output "$TMPFILE" "$SIGNATUREFILENAME" 2>&1) @@ -111,7 +130,7 @@ if [ $RET -ne 0 ]; then echo "gpg output:" echo "$GPGOUT"|sed 's/^/\t/g' - clean_up $SIGNATUREFILENAME $TMPFILE + clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE exit "$RET" fi @@ -131,7 +150,7 @@ FILES=$(awk '{print $2}' "$TMPFILE") for file in $FILES do echo "Downloading $file" - wget --quiet -N "$BASEDIR$file" + wget --quiet -N "$HOST1$BASEDIR$file" done #check hashes @@ -149,7 +168,7 @@ fi if [ -n "$2" ]; then echo "Clean up the binaries" - clean_up $FILES $SIGNATUREFILENAME $TMPFILE + clean_up $FILES $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE else echo "Keep the binaries in $WORKINGDIR" clean_up $TMPFILE diff --git a/src/base58.cpp b/src/base58.cpp index efa1beb1e4..17022a6bc1 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -110,7 +110,7 @@ std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend) std::string EncodeBase58(const std::vector<unsigned char>& vch) { - return EncodeBase58(&vch[0], &vch[0] + vch.size()); + return EncodeBase58(vch.data(), vch.data() + vch.size()); } bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet) @@ -160,7 +160,7 @@ void CBase58Data::SetData(const std::vector<unsigned char>& vchVersionIn, const vchVersion = vchVersionIn; vchData.resize(nSize); if (!vchData.empty()) - memcpy(&vchData[0], pdata, nSize); + memcpy(vchData.data(), pdata, nSize); } void CBase58Data::SetData(const std::vector<unsigned char>& vchVersionIn, const unsigned char* pbegin, const unsigned char* pend) @@ -180,8 +180,8 @@ bool CBase58Data::SetString(const char* psz, unsigned int nVersionBytes) vchVersion.assign(vchTemp.begin(), vchTemp.begin() + nVersionBytes); vchData.resize(vchTemp.size() - nVersionBytes); if (!vchData.empty()) - memcpy(&vchData[0], &vchTemp[nVersionBytes], vchData.size()); - memory_cleanse(&vchTemp[0], vchTemp.size()); + memcpy(vchData.data(), vchTemp.data() + nVersionBytes, vchData.size()); + memory_cleanse(vchTemp.data(), vchTemp.size()); return true; } @@ -262,7 +262,7 @@ CTxDestination CBitcoinAddress::Get() const if (!IsValid()) return CNoDestination(); uint160 id; - memcpy(&id, &vchData[0], 20); + memcpy(&id, vchData.data(), 20); if (vchVersion == Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)) return CKeyID(id); else if (vchVersion == Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS)) @@ -276,7 +276,7 @@ bool CBitcoinAddress::GetKeyID(CKeyID& keyID) const if (!IsValid() || vchVersion != Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)) return false; uint160 id; - memcpy(&id, &vchData[0], 20); + memcpy(&id, vchData.data(), 20); keyID = CKeyID(id); return true; } diff --git a/src/base58.h b/src/base58.h index 3998283bb1..4de5cc6ce5 100644 --- a/src/base58.h +++ b/src/base58.h @@ -148,7 +148,7 @@ public: K ret; if (vchData.size() == Size) { // If base58 encoded data does not hold an ext key, return a !IsValid() key - ret.Decode(&vchData[0]); + ret.Decode(vchData.data()); } return ret; } diff --git a/src/compressor.cpp b/src/compressor.cpp index 20c154fc1e..f4c12f38d2 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -93,7 +93,7 @@ bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector<unsigne script[0] = OP_DUP; script[1] = OP_HASH160; script[2] = 20; - memcpy(&script[3], &in[0], 20); + memcpy(&script[3], in.data(), 20); script[23] = OP_EQUALVERIFY; script[24] = OP_CHECKSIG; return true; @@ -101,7 +101,7 @@ bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector<unsigne script.resize(23); script[0] = OP_HASH160; script[1] = 20; - memcpy(&script[2], &in[0], 20); + memcpy(&script[2], in.data(), 20); script[22] = OP_EQUAL; return true; case 0x02: @@ -109,14 +109,14 @@ bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector<unsigne script.resize(35); script[0] = 33; script[1] = nSize; - memcpy(&script[2], &in[0], 32); + memcpy(&script[2], in.data(), 32); script[34] = OP_CHECKSIG; return true; case 0x04: case 0x05: unsigned char vch[33] = {}; vch[0] = nSize - 2; - memcpy(&vch[1], &in[0], 32); + memcpy(&vch[1], in.data(), 32); CPubKey pubkey(&vch[0], &vch[33]); if (!pubkey.Decompress()) return false; diff --git a/src/core_read.cpp b/src/core_read.cpp index dd9b5726a3..18d02fb913 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -27,7 +27,7 @@ CScript ParseScript(const std::string& s) if (mapOpNames.empty()) { - for (int op = 0; op <= OP_NOP10; op++) + for (int op = 0; op <= MAX_OPCODE; op++) { // Allow OP_RESERVED to get into mapOpNames if (op < OP_NOP && op != OP_RESERVED) diff --git a/src/hash.cpp b/src/hash.cpp index b361c90d16..5a15600be5 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -17,36 +17,34 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char { // The following is MurmurHash3 (x86_32), see http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp uint32_t h1 = nHashSeed; - if (vDataToHash.size() > 0) - { - const uint32_t c1 = 0xcc9e2d51; - const uint32_t c2 = 0x1b873593; + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; - const int nblocks = vDataToHash.size() / 4; + const int nblocks = vDataToHash.size() / 4; - //---------- - // body - const uint8_t* blocks = &vDataToHash[0] + nblocks * 4; + //---------- + // body + const uint8_t* blocks = vDataToHash.data(); - for (int i = -nblocks; i; i++) { - uint32_t k1 = ReadLE32(blocks + i*4); + for (int i = 0; i < nblocks; ++i) { + uint32_t k1 = ReadLE32(blocks + i*4); - k1 *= c1; - k1 = ROTL32(k1, 15); - k1 *= c2; + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; - h1 ^= k1; - h1 = ROTL32(h1, 13); - h1 = h1 * 5 + 0xe6546b64; - } + h1 ^= k1; + h1 = ROTL32(h1, 13); + h1 = h1 * 5 + 0xe6546b64; + } - //---------- - // tail - const uint8_t* tail = (const uint8_t*)(&vDataToHash[0] + nblocks * 4); + //---------- + // tail + const uint8_t* tail = vDataToHash.data() + nblocks * 4; - uint32_t k1 = 0; + uint32_t k1 = 0; - switch (vDataToHash.size() & 3) { + switch (vDataToHash.size() & 3) { case 3: k1 ^= tail[2] << 16; case 2: @@ -57,7 +55,6 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char k1 = ROTL32(k1, 15); k1 *= c2; h1 ^= k1; - } } //---------- diff --git a/src/init.cpp b/src/init.cpp index 672ef77e80..1e85642019 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -215,6 +215,19 @@ void Shutdown() fFeeEstimatesInitialized = false; } + // FlushStateToDisk generates a SetBestChain callback, which we should avoid missing + FlushStateToDisk(); + + // After there are no more peers/RPC left to give us new data which may generate + // CValidationInterface callbacks, flush them... + GetMainSignals().FlushBackgroundCallbacks(); + + // Any future callbacks will be dropped. This should absolutely be safe - if + // missing a callback results in an unrecoverable situation, unclean shutdown + // would too. The only reason to do the above flushes is to let the wallet catch + // up with our current chain to avoid any strange pruning edge cases and make + // next startup faster by avoiding rescan. + { LOCK(cs_main); if (pcoinsTip != NULL) { @@ -251,6 +264,7 @@ void Shutdown() } #endif UnregisterAllValidationInterfaces(); + GetMainSignals().UnregisterBackgroundSignalScheduler(); #ifdef ENABLE_WALLET for (CWalletRef pwallet : vpwallets) { delete pwallet; @@ -1203,6 +1217,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CScheduler::Function serviceLoop = boost::bind(&CScheduler::serviceQueue, &scheduler); threadGroup.create_thread(boost::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections * that the server is there and will be ready later). Warmup mode will @@ -1496,7 +1512,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) LogPrintf("Shutdown requested. Exiting.\n"); return false; } - LogPrintf(" block index %15dms\n", GetTimeMillis() - nStart); + if (fLoaded) { + LogPrintf(" block index %15dms\n", GetTimeMillis() - nStart); + } fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, CLIENT_VERSION); diff --git a/src/key.cpp b/src/key.cpp index 5a75647f1a..5a991fc1d2 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -138,7 +138,7 @@ CPrivKey CKey::GetPrivKey() const { size_t privkeylen; privkey.resize(279); privkeylen = 279; - ret = ec_privkey_export_der(secp256k1_context_sign, (unsigned char*)&privkey[0], &privkeylen, begin(), fCompressed ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); + ret = ec_privkey_export_der(secp256k1_context_sign, (unsigned char*) privkey.data(), &privkeylen, begin(), fCompressed ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); assert(ret); privkey.resize(privkeylen); return privkey; @@ -167,7 +167,7 @@ bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig, uint32_ secp256k1_ecdsa_signature sig; int ret = secp256k1_ecdsa_sign(secp256k1_context_sign, &sig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, test_case ? extra_entropy : NULL); assert(ret); - secp256k1_ecdsa_signature_serialize_der(secp256k1_context_sign, (unsigned char*)&vchSig[0], &nSigLen, &sig); + secp256k1_ecdsa_signature_serialize_der(secp256k1_context_sign, (unsigned char*)vchSig.data(), &nSigLen, &sig); vchSig.resize(nSigLen); return true; } @@ -202,7 +202,7 @@ bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) } bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck=false) { - if (!ec_privkey_import_der(secp256k1_context_sign, (unsigned char*)begin(), &privkey[0], privkey.size())) + if (!ec_privkey_import_der(secp256k1_context_sign, (unsigned char*)begin(), privkey.data(), privkey.size())) return false; fCompressed = vchPubKey.IsCompressed(); fValid = true; @@ -245,8 +245,8 @@ void CExtKey::SetMaster(const unsigned char *seed, unsigned int nSeedLen) { static const unsigned char hashkey[] = {'B','i','t','c','o','i','n',' ','s','e','e','d'}; std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(vout.data()); - key.Set(&vout[0], &vout[32], true); - memcpy(chaincode.begin(), &vout[32], 32); + key.Set(vout.data(), vout.data() + 32, true); + memcpy(chaincode.begin(), vout.data() + 32, 32); nDepth = 0; nChild = 0; memset(vchFingerprint, 0, sizeof(vchFingerprint)); diff --git a/src/net.cpp b/src/net.cpp index 301cf58b87..5bf3af7ea3 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2876,5 +2876,5 @@ uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const { std::vector<unsigned char> vchNetGroup(ad.GetGroup()); - return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(&vchNetGroup[0], vchNetGroup.size()).Finalize(); + return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 89f257c642..110e778fbd 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -573,7 +573,7 @@ std::vector<unsigned char> CService::GetKey() const { std::vector<unsigned char> vKey; vKey.resize(18); - memcpy(&vKey[0], ip, 16); + memcpy(vKey.data(), ip, 16); vKey[16] = port / 0x100; vKey[17] = port & 0x0FF; return vKey; diff --git a/src/policy/feerate.h b/src/policy/feerate.h index e82268b095..565da6c154 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -40,6 +40,7 @@ public: friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } + friend bool operator!=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK != b.nSatoshisPerK; } CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } std::string ToString() const; diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 771491770e..03fe11a0d8 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -16,6 +16,19 @@ static constexpr double INF_FEERATE = 1e99; +std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon) { + static const std::map<FeeEstimateHorizon, std::string> horizon_strings = { + {FeeEstimateHorizon::SHORT_HALFLIFE, "short"}, + {FeeEstimateHorizon::MED_HALFLIFE, "medium"}, + {FeeEstimateHorizon::LONG_HALFLIFE, "long"}, + }; + auto horizon_string = horizon_strings.find(horizon); + + if (horizon_string == horizon_strings.end()) return "unknown"; + + return horizon_string->second; +} + std::string StringForFeeReason(FeeReason reason) { static const std::map<FeeReason, std::string> fee_reason_strings = { {FeeReason::NONE, "None"}, @@ -36,6 +49,20 @@ std::string StringForFeeReason(FeeReason reason) { return reason_string->second; } +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) { + static const std::map<std::string, FeeEstimateMode> fee_modes = { + {"UNSET", FeeEstimateMode::UNSET}, + {"ECONOMICAL", FeeEstimateMode::ECONOMICAL}, + {"CONSERVATIVE", FeeEstimateMode::CONSERVATIVE}, + }; + auto mode = fee_modes.find(mode_string); + + if (mode == fee_modes.end()) return false; + + fee_estimate_mode = mode->second; + return true; +} + /** * We will instantiate an instance of this class to track transactions that were * included in a block. We will lump transactions into a bucket according to their @@ -671,7 +698,7 @@ CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThr break; } default: { - return CFeeRate(0); + throw std::out_of_range("CBlockPoicyEstimator::estimateRawFee unknown FeeEstimateHorizon"); } } @@ -690,6 +717,24 @@ CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThr return CFeeRate(median); } +unsigned int CBlockPolicyEstimator::HighestTargetTracked(FeeEstimateHorizon horizon) const +{ + switch (horizon) { + case FeeEstimateHorizon::SHORT_HALFLIFE: { + return shortStats->GetMaxConfirms(); + } + case FeeEstimateHorizon::MED_HALFLIFE: { + return feeStats->GetMaxConfirms(); + } + case FeeEstimateHorizon::LONG_HALFLIFE: { + return longStats->GetMaxConfirms(); + } + default: { + throw std::out_of_range("CBlockPoicyEstimator::HighestTargetTracked unknown FeeEstimateHorizon"); + } + } +} + unsigned int CBlockPolicyEstimator::BlockSpan() const { if (firstRecordedHeight == 0) return 0; diff --git a/src/policy/fees.h b/src/policy/fees.h index 2029ce3744..4c80371c5c 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -74,6 +74,8 @@ enum FeeEstimateHorizon { LONG_HALFLIFE = 2 }; +std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon); + /* Enumeration of reason for returned fee estimate */ enum class FeeReason { NONE, @@ -90,6 +92,15 @@ enum class FeeReason { std::string StringForFeeReason(FeeReason reason); +/* Used to determine type of fee estimation requested */ +enum class FeeEstimateMode { + UNSET, //! Use default settings based on other criteria + ECONOMICAL, //! Force estimateSmartFee to use non-conservative estimates + CONSERVATIVE, //! Force estimateSmartFee to use conservative estimates +}; + +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); + /* Used to return detailed information about a feerate bucket */ struct EstimatorBucket { @@ -197,7 +208,7 @@ public: * the closest target where one can be given. 'conservative' estimates are * valid over longer time horizons also. */ - CFeeRate estimateSmartFee(int confTarget, FeeCalculation *feeCalc, const CTxMemPool& pool, bool conservative = true) const; + CFeeRate estimateSmartFee(int confTarget, FeeCalculation *feeCalc, const CTxMemPool& pool, bool conservative) const; /** Return a specific fee estimate calculation with a given success * threshold and time horizon, and optionally return detailed data about @@ -214,6 +225,9 @@ public: /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ void FlushUnconfirmed(CTxMemPool& pool); + /** Calculation of highest target that estimates are tracked for */ + unsigned int HighestTargetTracked(FeeEstimateHorizon horizon) const; + private: unsigned int nBestSeenHeight; unsigned int firstRecordedHeight; diff --git a/src/pubkey.cpp b/src/pubkey.cpp index a16457ea4e..91af4e56f2 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -172,10 +172,7 @@ bool CPubKey::Verify(const uint256 &hash, const std::vector<unsigned char>& vchS if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, &(*this)[0], size())) { return false; } - if (vchSig.size() == 0) { - return false; - } - if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, &vchSig[0], vchSig.size())) { + if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, vchSig.data(), vchSig.size())) { return false; } /* libsecp256k1's ECDSA verification requires lower-S signatures, which have @@ -274,7 +271,7 @@ bool CExtPubKey::Derive(CExtPubKey &out, unsigned int _nChild) const { /* static */ bool CPubKey::CheckLowS(const std::vector<unsigned char>& vchSig) { secp256k1_ecdsa_signature sig; - if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, &vchSig[0], vchSig.size())) { + if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, vchSig.data(), vchSig.size())) { return false; } return (!secp256k1_ecdsa_signature_normalize(secp256k1_context_verify, NULL, &sig)); diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index af9a888d94..c19420beb5 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -490,6 +490,8 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) else nBytesInputs += 148; } + bool conservative_estimate = CalculateEstimateType(FeeEstimateMode::UNSET, coinControl->signalRbf); + // calculation if (nQuantity > 0) { @@ -510,7 +512,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nBytes -= 34; // Fee - nPayFee = CWallet::GetMinimumFee(nBytes, coinControl->nConfirmTarget, ::mempool, ::feeEstimator); + nPayFee = CWallet::GetMinimumFee(nBytes, coinControl->nConfirmTarget, ::mempool, ::feeEstimator, nullptr /* FeeCalculation */, false /* ignoreGlobalPayTxFee */, conservative_estimate); if (nPayAmount > 0) { @@ -585,7 +587,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (payTxFee.GetFeePerK() > 0) dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), payTxFee.GetFeePerK()) / 1000; else { - dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), ::feeEstimator.estimateSmartFee(coinControl->nConfirmTarget, NULL, ::mempool).GetFeePerK()) / 1000; + dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), ::feeEstimator.estimateSmartFee(coinControl->nConfirmTarget, NULL, ::mempool, conservative_estimate).GetFeePerK()) / 1000; } QString toolTip4 = tr("Can vary +/- %1 satoshi(s) per input.").arg(dFeeVary); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 12d2d0f31c..27634eb179 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -166,6 +166,8 @@ void SendCoinsDialog::setModel(WalletModel *_model) connect(ui->checkBoxMinimumFee, SIGNAL(stateChanged(int)), this, SLOT(updateFeeSectionControls())); connect(ui->checkBoxMinimumFee, SIGNAL(stateChanged(int)), this, SLOT(updateGlobalFeeVariables())); connect(ui->checkBoxMinimumFee, SIGNAL(stateChanged(int)), this, SLOT(coinControlUpdateLabels())); + connect(ui->optInRBF, SIGNAL(stateChanged(int)), this, SLOT(updateSmartFeeLabel())); + connect(ui->optInRBF, SIGNAL(stateChanged(int)), this, SLOT(coinControlUpdateLabels())); ui->customFee->setSingleStep(CWallet::GetRequiredFee(1000)); updateFeeSectionControls(); updateMinFeeLabel(); @@ -652,7 +654,8 @@ void SendCoinsDialog::updateSmartFeeLabel() int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; FeeCalculation feeCalc; - CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocksToConfirm, &feeCalc, ::mempool); + bool conservative_estimate = CalculateEstimateType(FeeEstimateMode::UNSET, ui->optInRBF->isChecked()); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocksToConfirm, &feeCalc, ::mempool, conservative_estimate); if (feeRate <= CFeeRate(0)) // not enough data => minfee { ui->labelSmartFee->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), @@ -827,6 +830,7 @@ void SendCoinsDialog::coinControlUpdateLabels() } else { CoinControlDialog::coinControl->nConfirmTarget = model->getDefaultConfirmTarget(); } + CoinControlDialog::coinControl->signalRbf = ui->optInRBF->isChecked(); for(int i = 0; i < ui->entries->count(); ++i) { diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 26dec3c610..fbad9e544a 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -12,6 +12,7 @@ #include "rpc/server.h" #include "rpcconsole.h" #include "test/testutil.h" +#include "test/test_bitcoin.h" #include "univalue.h" #include "util.h" @@ -35,8 +36,6 @@ void RPCNestedTests::rpcNestedTests() { // do some test setup // could be moved to a more generic place when we add more tests on QT level - const CChainParams& chainparams = Params(); - RegisterAllCoreRPCCommands(tableRPC); tableRPC.appendCommand("rpcNestedTest", &vRPCCommands[0]); ClearDatadirCache(); std::string path = QDir::tempPath().toStdString() + "/" + strprintf("test_bitcoin_qt_%lu_%i", (unsigned long)GetTime(), (int)(GetRand(100000))); @@ -44,15 +43,8 @@ void RPCNestedTests::rpcNestedTests() dir.mkpath("."); ForceSetArg("-datadir", path); //mempool.setSanityCheck(1.0); - pblocktree = new CBlockTreeDB(1 << 20, true); - pcoinsdbview = new CCoinsViewDB(1 << 23, true); - pcoinsTip = new CCoinsViewCache(pcoinsdbview); - InitBlockIndex(chainparams); - { - CValidationState state; - bool ok = ActivateBestChain(state, chainparams); - QVERIFY(ok); - } + + TestingSetup test; SetRPCWarmupFinished(); @@ -145,13 +137,5 @@ void RPCNestedTests::rpcNestedTests() QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(result, "rpcNestedTest(abc,,)"), std::runtime_error); //don't tollerate empty arguments when using , #endif - UnloadBlockIndex(); - delete pcoinsTip; - pcoinsTip = nullptr; - delete pcoinsdbview; - pcoinsdbview = nullptr; - delete pblocktree; - pblocktree = nullptr; - fs::remove_all(fs::path(path)); } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 7eff783fe8..60b55da3e7 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -19,6 +19,7 @@ #include "keystore.h" #include "validation.h" #include "net.h" // for g_connman +#include "policy/fees.h" #include "policy/rbf.h" #include "sync.h" #include "ui_interface.h" @@ -667,7 +668,7 @@ bool WalletModel::bumpFee(uint256 hash) std::unique_ptr<CFeeBumper> feeBump; { LOCK2(cs_main, wallet->cs_wallet); - feeBump.reset(new CFeeBumper(wallet, hash, nTxConfirmTarget, false, 0, true)); + feeBump.reset(new CFeeBumper(wallet, hash, nTxConfirmTarget, false, 0, true, FeeEstimateMode::UNSET)); } if (feeBump->getResult() != BumpFeeResult::OK) { diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index a3ea5390ee..775ad4b6c9 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -37,6 +37,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getnetworkhashps", 1, "height" }, { "sendtoaddress", 1, "amount" }, { "sendtoaddress", 4, "subtractfeefromamount" }, + { "sendtoaddress", 5 , "replaceable" }, + { "sendtoaddress", 6 , "conf_target" }, { "settxfee", 0, "amount" }, { "getreceivedbyaddress", 1, "minconf" }, { "getreceivedbyaccount", 1, "minconf" }, @@ -69,6 +71,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendmany", 1, "amounts" }, { "sendmany", 2, "minconf" }, { "sendmany", 4, "subtractfeefrom" }, + { "sendmany", 5 , "replaceable" }, + { "sendmany", 6 , "conf_target" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, { "createmultisig", 0, "nrequired" }, @@ -79,6 +83,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listunspent", 3, "include_unsafe" }, { "listunspent", 4, "query_options" }, { "getblock", 1, "verbosity" }, + { "getblock", 1, "verbose" }, { "getblockheader", 1, "verbose" }, { "getchaintxstats", 0, "nblocks" }, { "gettransaction", 1, "include_watchonly" }, @@ -112,7 +117,6 @@ static const CRPCConvertParam vRPCConvertParams[] = { "estimatesmartfee", 1, "conservative" }, { "estimaterawfee", 0, "nblocks" }, { "estimaterawfee", 1, "threshold" }, - { "estimaterawfee", 2, "horizon" }, { "prioritisetransaction", 1, "dummy" }, { "prioritisetransaction", 2, "fee_delta" }, { "setban", 2, "bantime" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index e50742f36e..046c85a76e 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -838,9 +838,9 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) UniValue estimaterawfee(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1|| request.params.size() > 3) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "estimaterawfee nblocks (threshold horizon)\n" + "estimaterawfee nblocks (threshold)\n" "\nWARNING: This interface is unstable and may disappear or change!\n" "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" " implementation of fee estimation. The parameters it can be called with\n" @@ -849,72 +849,95 @@ UniValue estimaterawfee(const JSONRPCRequest& request) "confirmation within nblocks blocks if possible. Uses virtual transaction size as defined\n" "in BIP 141 (witness data is discounted).\n" "\nArguments:\n" - "1. nblocks (numeric)\n" + "1. nblocks (numeric) Confirmation target in blocks (1 - 1008)\n" "2. threshold (numeric, optional) The proportion of transactions in a given feerate range that must have been\n" " confirmed within nblocks in order to consider those feerates as high enough and proceed to check\n" " lower buckets. Default: 0.95\n" - "3. horizon (numeric, optional) How long a history of estimates to consider. 0=short, 1=medium, 2=long.\n" - " Default: 1\n" "\nResult:\n" "{\n" - " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" - " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" - " \"scale\" : x, (numeric) The resolution of confirmation targets at this time horizon\n" - " \"pass\" : { (json object) information about the lowest range of feerates to succeed in meeting the threshold\n" - " \"startrange\" : x.x, (numeric) start of feerate range\n" - " \"endrange\" : x.x, (numeric) end of feerate range\n" - " \"withintarget\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed within target\n" - " \"totalconfirmed\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed at any point\n" - " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" - " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" - " }\n" - " \"fail\" : { ... } (json object) information about the highest range of feerates to fail to meet the threshold\n" + " \"short\" : { (json object, optional) estimate for short time horizon\n" + " \"feerate\" : x.x, (numeric, optional) estimate fee-per-kilobyte (in BTC)\n" + " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" + " \"scale\" : x, (numeric) The resolution of confirmation targets at this time horizon\n" + " \"pass\" : { (json object, optional) information about the lowest range of feerates to succeed in meeting the threshold\n" + " \"startrange\" : x.x, (numeric) start of feerate range\n" + " \"endrange\" : x.x, (numeric) end of feerate range\n" + " \"withintarget\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed within target\n" + " \"totalconfirmed\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed at any point\n" + " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" + " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" + " },\n" + " \"fail\" : { ... }, (json object, optional) information about the highest range of feerates to fail to meet the threshold\n" + " \"errors\": [ str... ] (json array of strings, optional) Errors encountered during processing\n" + " },\n" + " \"medium\" : { ... }, (json object, optional) estimate for medium time horizon\n" + " \"long\" : { ... } (json object) estimate for long time horizon\n" "}\n" "\n" - "A negative feerate is returned if no answer can be given.\n" + "Results are returned for any horizon which tracks blocks up to the confirmation target.\n" "\nExample:\n" - + HelpExampleCli("estimaterawfee", "6 0.9 1") + + HelpExampleCli("estimaterawfee", "6 0.9") ); RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM, UniValue::VNUM}, true); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); int nBlocks = request.params[0].get_int(); + if (nBlocks < 1 || (unsigned int)nBlocks > ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid nblocks"); + } double threshold = 0.95; - if (!request.params[1].isNull()) + if (!request.params[1].isNull()) { threshold = request.params[1].get_real(); - FeeEstimateHorizon horizon = FeeEstimateHorizon::MED_HALFLIFE; - if (!request.params[2].isNull()) { - int horizonInt = request.params[2].get_int(); - if (horizonInt < 0 || horizonInt > 2) { - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid horizon for fee estimates"); - } else { - horizon = (FeeEstimateHorizon)horizonInt; - } } + if (threshold < 0 || threshold > 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid threshold"); + } + UniValue result(UniValue::VOBJ); - CFeeRate feeRate; - EstimationResult buckets; - feeRate = ::feeEstimator.estimateRawFee(nBlocks, threshold, horizon, &buckets); - result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); - result.push_back(Pair("decay", buckets.decay)); - result.push_back(Pair("scale", (int)buckets.scale)); - UniValue passbucket(UniValue::VOBJ); - passbucket.push_back(Pair("startrange", round(buckets.pass.start))); - passbucket.push_back(Pair("endrange", round(buckets.pass.end))); - passbucket.push_back(Pair("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0)); - passbucket.push_back(Pair("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0)); - passbucket.push_back(Pair("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0)); - passbucket.push_back(Pair("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0)); - result.push_back(Pair("pass", passbucket)); - UniValue failbucket(UniValue::VOBJ); - failbucket.push_back(Pair("startrange", round(buckets.fail.start))); - failbucket.push_back(Pair("endrange", round(buckets.fail.end))); - failbucket.push_back(Pair("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0)); - failbucket.push_back(Pair("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0)); - failbucket.push_back(Pair("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0)); - failbucket.push_back(Pair("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0)); - result.push_back(Pair("fail", failbucket)); + for (FeeEstimateHorizon horizon : {FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE}) { + CFeeRate feeRate; + EstimationResult buckets; + + // Only output results for horizons which track the target + if ((unsigned int)nBlocks > ::feeEstimator.HighestTargetTracked(horizon)) continue; + + feeRate = ::feeEstimator.estimateRawFee(nBlocks, threshold, horizon, &buckets); + UniValue horizon_result(UniValue::VOBJ); + UniValue errors(UniValue::VARR); + UniValue passbucket(UniValue::VOBJ); + passbucket.push_back(Pair("startrange", round(buckets.pass.start))); + passbucket.push_back(Pair("endrange", round(buckets.pass.end))); + passbucket.push_back(Pair("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0)); + passbucket.push_back(Pair("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0)); + passbucket.push_back(Pair("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0)); + passbucket.push_back(Pair("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0)); + UniValue failbucket(UniValue::VOBJ); + failbucket.push_back(Pair("startrange", round(buckets.fail.start))); + failbucket.push_back(Pair("endrange", round(buckets.fail.end))); + failbucket.push_back(Pair("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0)); + failbucket.push_back(Pair("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0)); + failbucket.push_back(Pair("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0)); + failbucket.push_back(Pair("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0)); + + // CFeeRate(0) is used to indicate error as a return value from estimateRawFee + if (feeRate != CFeeRate(0)) { + horizon_result.push_back(Pair("feerate", ValueFromAmount(feeRate.GetFeePerK()))); + horizon_result.push_back(Pair("decay", buckets.decay)); + horizon_result.push_back(Pair("scale", (int)buckets.scale)); + horizon_result.push_back(Pair("pass", passbucket)); + // buckets.fail.start == -1 indicates that all buckets passed, there is no fail bucket to output + if (buckets.fail.start != -1) horizon_result.push_back(Pair("fail", failbucket)); + } else { + // Output only information that is still meaningful in the event of error + horizon_result.push_back(Pair("decay", buckets.decay)); + horizon_result.push_back(Pair("scale", (int)buckets.scale)); + horizon_result.push_back(Pair("fail", failbucket)); + errors.push_back("Insufficient data or no feerate found which meets threshold"); + horizon_result.push_back(Pair("errors",errors)); + } + result.push_back(Pair(StringForFeeEstimateHorizon(horizon), horizon_result)); + } return result; } @@ -932,7 +955,7 @@ static const CRPCCommand commands[] = { "util", "estimatefee", &estimatefee, true, {"nblocks"} }, { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks", "conservative"} }, - { "hidden", "estimaterawfee", &estimaterawfee, true, {"nblocks", "threshold", "horizon"} }, + { "hidden", "estimaterawfee", &estimaterawfee, true, {"nblocks", "threshold"} }, }; void RegisterMiningRPCCommands(CRPCTable &t) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index ef19e481c2..fcbbe1ceed 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -174,6 +174,14 @@ UniValue validateaddress(const JSONRPCRequest& request) " \"ismine\" : true|false, (boolean) If the address is yours or not\n" " \"iswatchonly\" : true|false, (boolean) If the address is watchonly\n" " \"isscript\" : true|false, (boolean) If the key is a script\n" + " \"script\" : \"type\" (string, optional) The output script type. Possible types: nonstandard, pubkey, pubkeyhash, scripthash, multisig, nulldata, witness_v0_keyhash, witness_v0_scripthash\n" + " \"hex\" : \"hex\", (string, optional) The redeemscript for the p2sh address\n" + " \"addresses\" (string, optional) Array of addresses associated with the known redeemscript\n" + " [\n" + " \"address\"\n" + " ,...\n" + " ]\n" + " \"sigsrequired\" : xxxxx (numeric, optional) Number of signatures required to spend multisig output\n" " \"pubkey\" : \"publickeyhex\", (string) The hex value of the raw public key\n" " \"iscompressed\" : true|false, (boolean) If the address is compressed\n" " \"account\" : \"account\" (string) DEPRECATED. The account associated with the address, \"\" is the default account\n" diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 923ba2c231..36a6d5110d 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -139,3 +139,69 @@ size_t CScheduler::getQueueInfo(boost::chrono::system_clock::time_point &first, } return result; } + +bool CScheduler::AreThreadsServicingQueue() const { + return nThreadsServicingQueue; +} + + +void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() { + { + LOCK(m_cs_callbacks_pending); + // Try to avoid scheduling too many copies here, but if we + // accidentally have two ProcessQueue's scheduled at once its + // not a big deal. + if (m_are_callbacks_running) return; + if (m_callbacks_pending.empty()) return; + } + m_pscheduler->schedule(std::bind(&SingleThreadedSchedulerClient::ProcessQueue, this)); +} + +void SingleThreadedSchedulerClient::ProcessQueue() { + std::function<void (void)> callback; + { + LOCK(m_cs_callbacks_pending); + if (m_are_callbacks_running) return; + if (m_callbacks_pending.empty()) return; + m_are_callbacks_running = true; + + callback = std::move(m_callbacks_pending.front()); + m_callbacks_pending.pop_front(); + } + + // RAII the setting of fCallbacksRunning and calling MaybeScheduleProcessQueue + // to ensure both happen safely even if callback() throws. + struct RAIICallbacksRunning { + SingleThreadedSchedulerClient* instance; + RAIICallbacksRunning(SingleThreadedSchedulerClient* _instance) : instance(_instance) {} + ~RAIICallbacksRunning() { + { + LOCK(instance->m_cs_callbacks_pending); + instance->m_are_callbacks_running = false; + } + instance->MaybeScheduleProcessQueue(); + } + } raiicallbacksrunning(this); + + callback(); +} + +void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void (void)> func) { + assert(m_pscheduler); + + { + LOCK(m_cs_callbacks_pending); + m_callbacks_pending.emplace_back(std::move(func)); + } + MaybeScheduleProcessQueue(); +} + +void SingleThreadedSchedulerClient::EmptyQueue() { + assert(!m_pscheduler->AreThreadsServicingQueue()); + bool should_continue = true; + while (should_continue) { + ProcessQueue(); + LOCK(m_cs_callbacks_pending); + should_continue = !m_callbacks_pending.empty(); + } +} diff --git a/src/scheduler.h b/src/scheduler.h index 27412a15b4..0365d668b2 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -14,6 +14,8 @@ #include <boost/thread.hpp> #include <map> +#include "sync.h" + // // Simple class for background tasks that should be run // periodically or once "after a while" @@ -41,7 +43,7 @@ public: typedef std::function<void(void)> Function; // Call func at/after time t - void schedule(Function f, boost::chrono::system_clock::time_point t); + void schedule(Function f, boost::chrono::system_clock::time_point t=boost::chrono::system_clock::now()); // Convenience method: call f once deltaSeconds from now void scheduleFromNow(Function f, int64_t deltaMilliSeconds); @@ -69,6 +71,9 @@ public: size_t getQueueInfo(boost::chrono::system_clock::time_point &first, boost::chrono::system_clock::time_point &last) const; + // Returns true if there are threads actively running in serviceQueue() + bool AreThreadsServicingQueue() const; + private: std::multimap<boost::chrono::system_clock::time_point, Function> taskQueue; boost::condition_variable newTaskScheduled; @@ -79,4 +84,30 @@ private: bool shouldStop() { return stopRequested || (stopWhenEmpty && taskQueue.empty()); } }; +/** + * Class used by CScheduler clients which may schedule multiple jobs + * which are required to be run serially. Does not require such jobs + * to be executed on the same thread, but no two jobs will be executed + * at the same time. + */ +class SingleThreadedSchedulerClient { +private: + CScheduler *m_pscheduler; + + CCriticalSection m_cs_callbacks_pending; + std::list<std::function<void (void)>> m_callbacks_pending; + bool m_are_callbacks_running = false; + + void MaybeScheduleProcessQueue(); + void ProcessQueue(); + +public: + SingleThreadedSchedulerClient(CScheduler *pschedulerIn) : m_pscheduler(pschedulerIn) {} + void AddToProcessQueue(std::function<void (void)> func); + + // Processes all remaining queue members on the calling thread, blocking until queue is empty + // Must be called after the CScheduler has no remaining processing threads! + void EmptyQueue(); +}; + #endif diff --git a/src/serialize.h b/src/serialize.h index e82ddf2c5a..8b86a07a76 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -450,7 +450,7 @@ public: } string.resize(size); if (size != 0) - s.read((char*)&string[0], size); + s.read((char*)string.data(), size); } template<typename Stream> @@ -458,7 +458,7 @@ public: { WriteCompactSize(s, string.size()); if (!string.empty()) - s.write((char*)&string[0], string.size()); + s.write((char*)string.data(), string.size()); } }; @@ -556,7 +556,7 @@ void Serialize(Stream& os, const std::basic_string<C>& str) { WriteCompactSize(os, str.size()); if (!str.empty()) - os.write((char*)&str[0], str.size() * sizeof(str[0])); + os.write((char*)str.data(), str.size() * sizeof(C)); } template<typename Stream, typename C> @@ -565,7 +565,7 @@ void Unserialize(Stream& is, std::basic_string<C>& str) unsigned int nSize = ReadCompactSize(is); str.resize(nSize); if (nSize != 0) - is.read((char*)&str[0], nSize * sizeof(str[0])); + is.read((char*)str.data(), nSize * sizeof(C)); } @@ -578,7 +578,7 @@ void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&) { WriteCompactSize(os, v.size()); if (!v.empty()) - os.write((char*)&v[0], v.size() * sizeof(T)); + os.write((char*)v.data(), v.size() * sizeof(T)); } template<typename Stream, unsigned int N, typename T, typename V> @@ -646,7 +646,7 @@ void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char& { WriteCompactSize(os, v.size()); if (!v.empty()) - os.write((char*)&v[0], v.size() * sizeof(T)); + os.write((char*)v.data(), v.size() * sizeof(T)); } template<typename Stream, typename T, typename A, typename V> diff --git a/src/streams.h b/src/streams.h index 8dc5a19ead..245fb9cd8f 100644 --- a/src/streams.h +++ b/src/streams.h @@ -389,7 +389,7 @@ public: { // Special case: stream << stream concatenates like stream += stream if (!vch.empty()) - s.write((char*)&vch[0], vch.size() * sizeof(vch[0])); + s.write((char*)vch.data(), vch.size() * sizeof(value_type)); } template<typename T> diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index 0390d6806d..698e898231 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -240,7 +240,7 @@ ["0", "IF NOP10 ENDIF 1", "P2SH,STRICTENC,DISCOURAGE_UPGRADABLE_NOPS", "OK", "Discouraged NOPs are allowed if not executed"], -["0", "IF 0xba ELSE 1 ENDIF", "P2SH,STRICTENC", "OK", "opcodes above NOP10 invalid if executed"], +["0", "IF 0xba ELSE 1 ENDIF", "P2SH,STRICTENC", "OK", "opcodes above MAX_OPCODE invalid if executed"], ["0", "IF 0xbb ELSE 1 ENDIF", "P2SH,STRICTENC", "OK"], ["0", "IF 0xbc ELSE 1 ENDIF", "P2SH,STRICTENC", "OK"], ["0", "IF 0xbd ELSE 1 ENDIF", "P2SH,STRICTENC", "OK"], @@ -878,7 +878,7 @@ "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS", "Discouraged NOP10 in redeemScript"], ["0x50","1", "P2SH,STRICTENC", "BAD_OPCODE", "opcode 0x50 is reserved"], -["1", "IF 0xba ELSE 1 ENDIF", "P2SH,STRICTENC", "BAD_OPCODE", "opcodes above NOP10 invalid if executed"], +["1", "IF 0xba ELSE 1 ENDIF", "P2SH,STRICTENC", "BAD_OPCODE", "opcodes above MAX_OPCODE invalid if executed"], ["1", "IF 0xbb ELSE 1 ENDIF", "P2SH,STRICTENC", "BAD_OPCODE"], ["1", "IF 0xbc ELSE 1 ENDIF", "P2SH,STRICTENC", "BAD_OPCODE"], ["1", "IF 0xbd ELSE 1 ENDIF", "P2SH,STRICTENC", "BAD_OPCODE"], @@ -1001,7 +1001,7 @@ ["1","RESERVED", "P2SH,STRICTENC", "BAD_OPCODE", "OP_RESERVED is reserved"], ["1","RESERVED1", "P2SH,STRICTENC", "BAD_OPCODE", "OP_RESERVED1 is reserved"], ["1","RESERVED2", "P2SH,STRICTENC", "BAD_OPCODE", "OP_RESERVED2 is reserved"], -["1","0xba", "P2SH,STRICTENC", "BAD_OPCODE", "0xba == OP_NOP10 + 1"], +["1","0xba", "P2SH,STRICTENC", "BAD_OPCODE", "0xba == MAX_OPCODE + 1"], ["2147483648", "1ADD 1", "P2SH,STRICTENC", "UNKNOWN_ERROR", "We cannot do math on 5-byte integers"], ["2147483648", "NEGATE 1", "P2SH,STRICTENC", "UNKNOWN_ERROR", "We cannot do math on 5-byte integers"], diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 6bfd315647..8cdd392109 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -184,8 +184,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateRawFee(i, 0.85, FeeEstimateHorizon::MED_HALFLIFE).GetFeePerK()); - BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool, true).GetFeePerK() >= feeEst.estimateRawFee(i, 0.85, FeeEstimateHorizon::MED_HALFLIFE).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool, true).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 579e96524c..3ba81ed17b 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -62,6 +62,12 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha pathTemp = GetTempPath() / strprintf("test_bitcoin_%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(100000))); fs::create_directories(pathTemp); ForceSetArg("-datadir", pathTemp.string()); + + // Note that because we don't bother running a scheduler thread here, + // callbacks via CValidationInterface are unreliable, but that's OK, + // our unit tests aren't testing multiple parts of the code at once. + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + mempool.setSanityCheck(1.0); pblocktree = new CBlockTreeDB(1 << 20, true); pcoinsdbview = new CCoinsViewDB(1 << 23, true); @@ -88,6 +94,8 @@ TestingSetup::~TestingSetup() UnregisterNodeSignals(GetNodeSignals()); threadGroup.interrupt_all(); threadGroup.join_all(); + GetMainSignals().FlushBackgroundCallbacks(); + GetMainSignals().UnregisterBackgroundSignalScheduler(); UnloadBlockIndex(); delete pcoinsTip; delete pcoinsdbview; diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index c9e4a3427f..dd3b13c8c8 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -10,6 +10,7 @@ #include "key.h" #include "pubkey.h" #include "random.h" +#include "scheduler.h" #include "txdb.h" #include "txmempool.h" @@ -53,6 +54,7 @@ struct TestingSetup: public BasicTestingSetup { fs::path pathTemp; boost::thread_group threadGroup; CConnman* connman; + CScheduler scheduler; TestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~TestingSetup(); diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 3665e7e770..ac13f73e70 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -662,7 +662,7 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro // _conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), boost::bind(&TorController::auth_cb, this, _1, _2)); cookie = std::vector<uint8_t>(status_cookie.second.begin(), status_cookie.second.end()); clientNonce = std::vector<uint8_t>(TOR_NONCE_SIZE, 0); - GetRandBytes(&clientNonce[0], TOR_NONCE_SIZE); + GetRandBytes(clientNonce.data(), TOR_NONCE_SIZE); _conn.Command("AUTHCHALLENGE SAFECOOKIE " + HexStr(clientNonce), boost::bind(&TorController::authchallenge_cb, this, _1, _2)); } else { if (status_cookie.first) { diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 93abaec04b..2fabb8cf7a 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -228,7 +228,7 @@ std::vector<unsigned char> DecodeBase64(const char* p, bool* pfInvalid) std::string DecodeBase64(const std::string& str) { std::vector<unsigned char> vchRet = DecodeBase64(str.c_str()); - return (vchRet.size() == 0) ? std::string() : std::string((const char*)&vchRet[0], vchRet.size()); + return std::string((const char*)vchRet.data(), vchRet.size()); } std::string EncodeBase32(const unsigned char* pch, size_t len) @@ -415,7 +415,7 @@ std::vector<unsigned char> DecodeBase32(const char* p, bool* pfInvalid) std::string DecodeBase32(const std::string& str) { std::vector<unsigned char> vchRet = DecodeBase32(str.c_str()); - return (vchRet.size() == 0) ? std::string() : std::string((const char*)&vchRet[0], vchRet.size()); + return std::string((const char*)vchRet.data(), vchRet.size()); } static bool ParsePrechecks(const std::string& str) diff --git a/src/validation.cpp b/src/validation.cpp index 6aa50a4da0..16d11529b0 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2890,7 +2890,7 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc if (consensusParams.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout != 0) { if (commitpos == -1) { uint256 witnessroot = BlockWitnessMerkleRoot(block, NULL); - CHash256().Write(witnessroot.begin(), 32).Write(&ret[0], 32).Finalize(witnessroot.begin()); + CHash256().Write(witnessroot.begin(), 32).Write(ret.data(), 32).Finalize(witnessroot.begin()); CTxOut out; out.nValue = 0; out.scriptPubKey.resize(38); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index be2f20b863..bf20d606f8 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -4,46 +4,123 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "validationinterface.h" +#include "init.h" +#include "scheduler.h" +#include "sync.h" +#include "util.h" + +#include <list> +#include <atomic> + +#include <boost/signals2/signal.hpp> + +struct MainSignalsInstance { + boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; + boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; + boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef>&)> BlockConnected; + boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; + boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; + boost::signals2::signal<void (const uint256 &)> Inventory; + boost::signals2::signal<void (int64_t nBestBlockTime, CConnman* connman)> Broadcast; + boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; + boost::signals2::signal<void (const CBlockIndex *, const std::shared_ptr<const CBlock>&)> NewPoWValidBlock; + + // We are not allowed to assume the scheduler only runs in one thread, + // but must ensure all callbacks happen in-order, so we end up creating + // our own queue here :( + SingleThreadedSchedulerClient m_schedulerClient; + + MainSignalsInstance(CScheduler *pscheduler) : m_schedulerClient(pscheduler) {} +}; static CMainSignals g_signals; +void CMainSignals::RegisterBackgroundSignalScheduler(CScheduler& scheduler) { + assert(!m_internals); + m_internals.reset(new MainSignalsInstance(&scheduler)); +} + +void CMainSignals::UnregisterBackgroundSignalScheduler() { + m_internals.reset(nullptr); +} + +void CMainSignals::FlushBackgroundCallbacks() { + m_internals->m_schedulerClient.EmptyQueue(); +} + CMainSignals& GetMainSignals() { return g_signals; } void RegisterValidationInterface(CValidationInterface* pwalletIn) { - g_signals.UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); - g_signals.TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); - g_signals.BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); - g_signals.BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); - g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); - g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); - g_signals.BlockChecked.connect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); - g_signals.NewPoWValidBlock.connect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); + g_signals.m_internals->UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); + g_signals.m_internals->TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); + g_signals.m_internals->BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); + g_signals.m_internals->BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.m_internals->SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); + g_signals.m_internals->Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); + g_signals.m_internals->Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); + g_signals.m_internals->BlockChecked.connect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); + g_signals.m_internals->NewPoWValidBlock.connect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterValidationInterface(CValidationInterface* pwalletIn) { - g_signals.BlockChecked.disconnect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); - g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); - g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); - g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); - g_signals.BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); - g_signals.BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); - g_signals.UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); - g_signals.NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); + g_signals.m_internals->BlockChecked.disconnect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); + g_signals.m_internals->Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); + g_signals.m_internals->Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); + g_signals.m_internals->SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); + g_signals.m_internals->TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); + g_signals.m_internals->BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); + g_signals.m_internals->BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.m_internals->UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); + g_signals.m_internals->NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterAllValidationInterfaces() { - g_signals.BlockChecked.disconnect_all_slots(); - g_signals.Broadcast.disconnect_all_slots(); - g_signals.Inventory.disconnect_all_slots(); - g_signals.SetBestChain.disconnect_all_slots(); - g_signals.TransactionAddedToMempool.disconnect_all_slots(); - g_signals.BlockConnected.disconnect_all_slots(); - g_signals.BlockDisconnected.disconnect_all_slots(); - g_signals.UpdatedBlockTip.disconnect_all_slots(); - g_signals.NewPoWValidBlock.disconnect_all_slots(); + g_signals.m_internals->BlockChecked.disconnect_all_slots(); + g_signals.m_internals->Broadcast.disconnect_all_slots(); + g_signals.m_internals->Inventory.disconnect_all_slots(); + g_signals.m_internals->SetBestChain.disconnect_all_slots(); + g_signals.m_internals->TransactionAddedToMempool.disconnect_all_slots(); + g_signals.m_internals->BlockConnected.disconnect_all_slots(); + g_signals.m_internals->BlockDisconnected.disconnect_all_slots(); + g_signals.m_internals->UpdatedBlockTip.disconnect_all_slots(); + g_signals.m_internals->NewPoWValidBlock.disconnect_all_slots(); +} + +void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { + m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); +} + +void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { + m_internals->TransactionAddedToMempool(ptx); +} + +void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { + m_internals->BlockConnected(pblock, pindex, vtxConflicted); +} + +void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock> &pblock) { + m_internals->BlockDisconnected(pblock); +} + +void CMainSignals::SetBestChain(const CBlockLocator &locator) { + m_internals->SetBestChain(locator); +} + +void CMainSignals::Inventory(const uint256 &hash) { + m_internals->Inventory(hash); +} + +void CMainSignals::Broadcast(int64_t nBestBlockTime, CConnman* connman) { + m_internals->Broadcast(nBestBlockTime, connman); +} + +void CMainSignals::BlockChecked(const CBlock& block, const CValidationState& state) { + m_internals->BlockChecked(block, state); +} + +void CMainSignals::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock> &block) { + m_internals->NewPoWValidBlock(pindex, block); } diff --git a/src/validationinterface.h b/src/validationinterface.h index 17545018df..568da66df2 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -6,7 +6,6 @@ #ifndef BITCOIN_VALIDATIONINTERFACE_H #define BITCOIN_VALIDATIONINTERFACE_H -#include <boost/signals2/signal.hpp> #include <memory> #include "primitives/transaction.h" // CTransaction(Ref) @@ -20,6 +19,7 @@ class CReserveScript; class CValidationInterface; class CValidationState; class uint256; +class CScheduler; // These functions dispatch to one or all registered wallets @@ -32,49 +32,66 @@ void UnregisterAllValidationInterfaces(); class CValidationInterface { protected: - virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} - virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} - virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} - virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} - virtual void SetBestChain(const CBlockLocator &locator) {} - virtual void Inventory(const uint256 &hash) {} - virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} - virtual void BlockChecked(const CBlock&, const CValidationState&) {} - virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {}; - friend void ::RegisterValidationInterface(CValidationInterface*); - friend void ::UnregisterValidationInterface(CValidationInterface*); - friend void ::UnregisterAllValidationInterfaces(); -}; - -struct CMainSignals { /** Notifies listeners of updated block chain tip */ - boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; + virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} /** Notifies listeners of a transaction having been added to mempool. */ - boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; + virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} /** * Notifies listeners of a block being connected. * Provides a vector of transactions evicted from the mempool as a result. */ - boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &)> BlockConnected; + virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} /** Notifies listeners of a block being disconnected */ - boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; - /** Notifies listeners of a new active block chain. */ - boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; + virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} + /** Notifies listeners of the new active block chain on-disk. */ + virtual void SetBestChain(const CBlockLocator &locator) {} /** Notifies listeners about an inventory item being seen on the network. */ - boost::signals2::signal<void (const uint256 &)> Inventory; + virtual void Inventory(const uint256 &hash) {} /** Tells listeners to broadcast their data. */ - boost::signals2::signal<void (int64_t nBestBlockTime, CConnman* connman)> Broadcast; + virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} /** * Notifies listeners of a block validation result. * If the provided CValidationState IsValid, the provided block * is guaranteed to be the current best block at the time the * callback was generated (not necessarily now) */ - boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; + virtual void BlockChecked(const CBlock&, const CValidationState&) {} /** * Notifies listeners that a block which builds directly on our current tip * has been received and connected to the headers tree, though not validated yet */ - boost::signals2::signal<void (const CBlockIndex *, const std::shared_ptr<const CBlock>&)> NewPoWValidBlock; + virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {}; + friend void ::RegisterValidationInterface(CValidationInterface*); + friend void ::UnregisterValidationInterface(CValidationInterface*); + friend void ::UnregisterAllValidationInterfaces(); +}; + +struct MainSignalsInstance; +class CMainSignals { +private: + std::unique_ptr<MainSignalsInstance> m_internals; + + friend void ::RegisterValidationInterface(CValidationInterface*); + friend void ::UnregisterValidationInterface(CValidationInterface*); + friend void ::UnregisterAllValidationInterfaces(); + +public: + /** Register a CScheduler to give callbacks which should run in the background (may only be called once) */ + void RegisterBackgroundSignalScheduler(CScheduler& scheduler); + /** Unregister a CScheduler to give callbacks which should run in the background - these callbacks will now be dropped! */ + void UnregisterBackgroundSignalScheduler(); + /** Call any remaining callbacks on the calling thread */ + void FlushBackgroundCallbacks(); + + void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); + void TransactionAddedToMempool(const CTransactionRef &); + void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &); + void BlockDisconnected(const std::shared_ptr<const CBlock> &); + void UpdatedTransaction(const uint256 &); + void SetBestChain(const CBlockLocator &); + void Inventory(const uint256 &); + void Broadcast(int64_t nBestBlockTime, CConnman* connman); + void BlockChecked(const CBlock&, const CValidationState&); + void NewPoWValidBlock(const CBlockIndex *, const std::shared_ptr<const CBlock>&); }; CMainSignals& GetMainSignals(); diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index cb4719ae90..bdd01bec12 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_COINCONTROL_H #include "policy/feerate.h" +#include "policy/fees.h" #include "primitives/transaction.h" #include "wallet/wallet.h" @@ -26,6 +27,8 @@ public: int nConfirmTarget; //! Signal BIP-125 replace by fee. bool signalRbf; + //! Fee estimation mode to control arguments to estimateSmartFee + FeeEstimateMode m_fee_mode; CCoinControl() { @@ -42,6 +45,7 @@ public: fOverrideFeeRate = false; nConfirmTarget = 0; signalRbf = fWalletRbf; + m_fee_mode = FeeEstimateMode::UNSET; } bool HasSelected() const diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 6a9e6cf9ff..607ecf4182 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -66,7 +66,7 @@ bool CFeeBumper::preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx return true; } -CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable) +CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable, FeeEstimateMode fee_mode) : txid(std::move(txidIn)), nOldFee(0), @@ -165,7 +165,8 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf nNewFee = totalFee; nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); } else { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, nullptr, ignoreGlobalPayTxFee); + bool conservative_estimate = CalculateEstimateType(fee_mode, newTxReplaceable); + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, nullptr /* FeeCalculation */, ignoreGlobalPayTxFee, conservative_estimate); nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); // New fee rate must be at least old rate + minimum incremental relay rate diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index fc32316704..11e2f5f953 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -10,6 +10,7 @@ class CWallet; class CWalletTx; class uint256; +enum class FeeEstimateMode; enum class BumpFeeResult { @@ -24,7 +25,7 @@ enum class BumpFeeResult class CFeeBumper { public: - CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable); + CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable, FeeEstimateMode fee_mode); BumpFeeResult getResult() const { return currentResult; } const std::vector<std::string>& getErrors() const { return vErrors; } CAmount getOldFee() const { return nOldFee; } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 867ccd4244..5f72e3b6f5 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -356,7 +356,7 @@ UniValue getaddressesbyaccount(const JSONRPCRequest& request) return ret; } -static void SendMoney(CWallet * const pwallet, const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, CWalletTx& wtxNew) +static void SendMoney(CWallet * const pwallet, const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, CWalletTx& wtxNew, CCoinControl *coin_control = nullptr) { CAmount curBalance = pwallet->GetBalance(); @@ -382,7 +382,7 @@ static void SendMoney(CWallet * const pwallet, const CTxDestination &address, CA int nChangePosRet = -1; CRecipient recipient = {scriptPubKey, nValue, fSubtractFeeFromAmount}; vecSend.push_back(recipient); - if (!pwallet->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError)) { + if (!pwallet->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError, coin_control)) { if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); throw JSONRPCError(RPC_WALLET_ERROR, strError); @@ -401,9 +401,9 @@ UniValue sendtoaddress(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() < 2 || request.params.size() > 5) + if (request.fHelp || request.params.size() < 2 || request.params.size() > 8) throw std::runtime_error( - "sendtoaddress \"address\" amount ( \"comment\" \"comment_to\" subtractfeefromamount )\n" + "sendtoaddress \"address\" amount ( \"comment\" \"comment_to\" subtractfeefromamount replaceable conf_target \"estimate_mode\")\n" "\nSend an amount to a given address.\n" + HelpRequiringPassphrase(pwallet) + "\nArguments:\n" @@ -416,6 +416,12 @@ UniValue sendtoaddress(const JSONRPCRequest& request) " transaction, just kept in your wallet.\n" "5. subtractfeefromamount (boolean, optional, default=false) The fee will be deducted from the amount being sent.\n" " The recipient will receive less bitcoins than you enter in the amount field.\n" + "6. replaceable (boolean, optional) Allow this transaction to be replaced by a transaction with higher fees via BIP 125\n" + "7. conf_target (numeric, optional) Confirmation target (in blocks)\n" + "8. \"estimate_mode\" (string, optional, default=UNSET) The fee estimate mode, must be one of:\n" + " \"UNSET\"\n" + " \"ECONOMICAL\"\n" + " \"CONSERVATIVE\"\n" "\nResult:\n" "\"txid\" (string) The transaction id.\n" "\nExamples:\n" @@ -444,12 +450,29 @@ UniValue sendtoaddress(const JSONRPCRequest& request) wtx.mapValue["to"] = request.params[3].get_str(); bool fSubtractFeeFromAmount = false; - if (request.params.size() > 4) + if (request.params.size() > 4 && !request.params[4].isNull()) { fSubtractFeeFromAmount = request.params[4].get_bool(); + } + + CCoinControl coin_control; + if (request.params.size() > 5 && !request.params[5].isNull()) { + coin_control.signalRbf = request.params[5].get_bool(); + } + + if (request.params.size() > 6 && !request.params[6].isNull()) { + coin_control.nConfirmTarget = request.params[6].get_int(); + } + + if (request.params.size() > 7 && !request.params[7].isNull()) { + if (!FeeModeFromString(request.params[7].get_str(), coin_control.m_fee_mode)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); + } + } + EnsureWalletIsUnlocked(pwallet); - SendMoney(pwallet, address.Get(), nAmount, fSubtractFeeFromAmount, wtx); + SendMoney(pwallet, address.Get(), nAmount, fSubtractFeeFromAmount, wtx, &coin_control); return wtx.GetHash().GetHex(); } @@ -888,9 +911,9 @@ UniValue sendmany(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() < 2 || request.params.size() > 5) + if (request.fHelp || request.params.size() < 2 || request.params.size() > 8) throw std::runtime_error( - "sendmany \"fromaccount\" {\"address\":amount,...} ( minconf \"comment\" [\"address\",...] )\n" + "sendmany \"fromaccount\" {\"address\":amount,...} ( minconf \"comment\" [\"address\",...] replaceable conf_target \"estimate_mode\")\n" "\nSend multiple times. Amounts are double-precision floating point numbers." + HelpRequiringPassphrase(pwallet) + "\n" "\nArguments:\n" @@ -910,7 +933,13 @@ UniValue sendmany(const JSONRPCRequest& request) " \"address\" (string) Subtract fee from this address\n" " ,...\n" " ]\n" - "\nResult:\n" + "6. replaceable (boolean, optional) Allow this transaction to be replaced by a transaction with higher fees via BIP 125\n" + "7. conf_target (numeric, optional) Confirmation target (in blocks)\n" + "8. \"estimate_mode\" (string, optional, default=UNSET) The fee estimate mode, must be one of:\n" + " \"UNSET\"\n" + " \"ECONOMICAL\"\n" + " \"CONSERVATIVE\"\n" + "\nResult:\n" "\"txid\" (string) The transaction id for the send. Only 1 transaction is created regardless of \n" " the number of addresses.\n" "\nExamples:\n" @@ -942,9 +971,24 @@ UniValue sendmany(const JSONRPCRequest& request) wtx.mapValue["comment"] = request.params[3].get_str(); UniValue subtractFeeFromAmount(UniValue::VARR); - if (request.params.size() > 4) + if (request.params.size() > 4 && !request.params[4].isNull()) subtractFeeFromAmount = request.params[4].get_array(); + CCoinControl coin_control; + if (request.params.size() > 5 && !request.params[5].isNull()) { + coin_control.signalRbf = request.params[5].get_bool(); + } + + if (request.params.size() > 6 && !request.params[6].isNull()) { + coin_control.nConfirmTarget = request.params[6].get_int(); + } + + if (request.params.size() > 7 && !request.params[7].isNull()) { + if (!FeeModeFromString(request.params[7].get_str(), coin_control.m_fee_mode)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); + } + } + std::set<CBitcoinAddress> setAddress; std::vector<CRecipient> vecSend; @@ -989,7 +1033,7 @@ UniValue sendmany(const JSONRPCRequest& request) CAmount nFeeRequired = 0; int nChangePosRet = -1; std::string strFailReason; - bool fCreated = pwallet->CreateTransaction(vecSend, wtx, keyChange, nFeeRequired, nChangePosRet, strFailReason); + bool fCreated = pwallet->CreateTransaction(vecSend, wtx, keyChange, nFeeRequired, nChangePosRet, strFailReason, &coin_control); if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); CValidationState state; @@ -2658,6 +2702,11 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) " [vout_index,...]\n" " \"replaceable\" (boolean, optional) Marks this transaction as BIP125 replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees\n" + " \"conf_target\" (numeric, optional) Confirmation target (in blocks)\n" + " \"estimate_mode\" (string, optional, default=UNSET) The fee estimate mode, must be one of:\n" + " \"UNSET\"\n" + " \"ECONOMICAL\"\n" + " \"CONSERVATIVE\"\n" " }\n" " for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}\n" "\nResult:\n" @@ -2710,6 +2759,8 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) {"feeRate", UniValueType()}, // will be checked below {"subtractFeeFromOutputs", UniValueType(UniValue::VARR)}, {"replaceable", UniValueType(UniValue::VBOOL)}, + {"conf_target", UniValueType(UniValue::VNUM)}, + {"estimate_mode", UniValueType(UniValue::VSTR)}, }, true, true); @@ -2746,6 +2797,14 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) if (options.exists("replaceable")) { coinControl.signalRbf = options["replaceable"].get_bool(); } + if (options.exists("conf_target")) { + coinControl.nConfirmTarget = options["conf_target"].get_int(); + } + if (options.exists("estimate_mode")) { + if (!FeeModeFromString(options["estimate_mode"].get_str(), coinControl.m_fee_mode)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); + } + } } } @@ -2823,6 +2882,10 @@ UniValue bumpfee(const JSONRPCRequest& request) " so the new transaction will not be explicitly bip-125 replaceable (though it may\n" " still be replaceable in practice, for example if it has unconfirmed ancestors which\n" " are replaceable).\n" + " \"estimate_mode\" (string, optional, default=UNSET) The fee estimate mode, must be one of:\n" + " \"UNSET\"\n" + " \"ECONOMICAL\"\n" + " \"CONSERVATIVE\"\n" " }\n" "\nResult:\n" "{\n" @@ -2845,6 +2908,7 @@ UniValue bumpfee(const JSONRPCRequest& request) int newConfirmTarget = nTxConfirmTarget; CAmount totalFee = 0; bool replaceable = true; + FeeEstimateMode fee_mode = FeeEstimateMode::UNSET; if (request.params.size() > 1) { UniValue options = request.params[1]; RPCTypeCheckObj(options, @@ -2852,6 +2916,7 @@ UniValue bumpfee(const JSONRPCRequest& request) {"confTarget", UniValueType(UniValue::VNUM)}, {"totalFee", UniValueType(UniValue::VNUM)}, {"replaceable", UniValueType(UniValue::VBOOL)}, + {"estimate_mode", UniValueType(UniValue::VSTR)}, }, true, true); @@ -2876,12 +2941,17 @@ UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("replaceable")) { replaceable = options["replaceable"].get_bool(); } + if (options.exists("estimate_mode")) { + if (!FeeModeFromString(options["estimate_mode"].get_str(), fee_mode)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); + } + } } LOCK2(cs_main, pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - CFeeBumper feeBump(pwallet, hash, newConfirmTarget, ignoreGlobalPayTxFee, totalFee, replaceable); + CFeeBumper feeBump(pwallet, hash, newConfirmTarget, ignoreGlobalPayTxFee, totalFee, replaceable, fee_mode); BumpFeeResult res = feeBump.getResult(); if (res != BumpFeeResult::OK) { @@ -3023,8 +3093,8 @@ static const CRPCCommand commands[] = { "wallet", "lockunspent", &lockunspent, true, {"unlock","transactions"} }, { "wallet", "move", &movecmd, false, {"fromaccount","toaccount","amount","minconf","comment"} }, { "wallet", "sendfrom", &sendfrom, false, {"fromaccount","toaddress","amount","minconf","comment","comment_to"} }, - { "wallet", "sendmany", &sendmany, false, {"fromaccount","amounts","minconf","comment","subtractfeefrom"} }, - { "wallet", "sendtoaddress", &sendtoaddress, false, {"address","amount","comment","comment_to","subtractfeefromamount"} }, + { "wallet", "sendmany", &sendmany, false, {"fromaccount","amounts","minconf","comment","subtractfeefrom","replaceable","conf_target","estimate_mode"} }, + { "wallet", "sendtoaddress", &sendtoaddress, false, {"address","amount","comment","comment_to","subtractfeefromamount","replaceable","conf_target","estimate_mode"} }, { "wallet", "setaccount", &setaccount, true, {"address","account"} }, { "wallet", "settxfee", &settxfee, true, {"amount"} }, { "wallet", "signmessage", &signmessage, true, {"address","message"} }, diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f5d9b33acc..5e9701c71c 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2569,7 +2569,43 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT std::vector<COutput> vAvailableCoins; AvailableCoins(vAvailableCoins, true, coinControl); + // Create change script that will be used if we need change + // TODO: pass in scriptChange instead of reservekey so + // change transaction isn't always pay-to-bitcoin-address + CScript scriptChange; + + // coin control: send change to custom address + if (coinControl && !boost::get<CNoDestination>(&coinControl->destChange)) + scriptChange = GetScriptForDestination(coinControl->destChange); + + // no coin control: send change to newly generated address + else + { + // Note: We use a new key here to keep it from being obvious which side is the change. + // The drawback is that by not reusing a previous key, the change may be lost if a + // backup is restored, if the backup doesn't have the new private key for the change. + // If we reused the old key, it would be possible to add code to look for and + // rediscover unknown transactions that were written with keys of ours to recover + // post-backup change. + + // Reserve a new key pair from key pool + CPubKey vchPubKey; + bool ret; + ret = reservekey.GetReservedKey(vchPubKey, true); + if (!ret) + { + strFailReason = _("Keypool ran out, please call keypoolrefill first"); + return false; + } + + scriptChange = GetScriptForDestination(vchPubKey.GetID()); + } + CTxOut change_prototype_txout(0, scriptChange); + size_t change_prototype_size = GetSerializeSize(change_prototype_txout, SER_DISK, 0); + nFeeRet = 0; + bool pick_new_inputs = true; + CAmount nValueIn = 0; // Start with no fee and loop until there is enough fee while (true) { @@ -2615,49 +2651,21 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT } // Choose coins to use - CAmount nValueIn = 0; - setCoins.clear(); - if (!SelectCoins(vAvailableCoins, nValueToSelect, setCoins, nValueIn, coinControl)) - { - strFailReason = _("Insufficient funds"); - return false; + if (pick_new_inputs) { + nValueIn = 0; + setCoins.clear(); + if (!SelectCoins(vAvailableCoins, nValueToSelect, setCoins, nValueIn, coinControl)) + { + strFailReason = _("Insufficient funds"); + return false; + } } const CAmount nChange = nValueIn - nValueToSelect; + if (nChange > 0) { // Fill a vout to ourself - // TODO: pass in scriptChange instead of reservekey so - // change transaction isn't always pay-to-bitcoin-address - CScript scriptChange; - - // coin control: send change to custom address - if (coinControl && !boost::get<CNoDestination>(&coinControl->destChange)) - scriptChange = GetScriptForDestination(coinControl->destChange); - - // no coin control: send change to newly generated address - else - { - // Note: We use a new key here to keep it from being obvious which side is the change. - // The drawback is that by not reusing a previous key, the change may be lost if a - // backup is restored, if the backup doesn't have the new private key for the change. - // If we reused the old key, it would be possible to add code to look for and - // rediscover unknown transactions that were written with keys of ours to recover - // post-backup change. - - // Reserve a new key pair from key pool - CPubKey vchPubKey; - bool ret; - ret = reservekey.GetReservedKey(vchPubKey, true); - if (!ret) - { - strFailReason = _("Keypool ran out, please call keypoolrefill first"); - return false; - } - - scriptChange = GetScriptForDestination(vchPubKey.GetID()); - } - CTxOut newTxOut(nChange, scriptChange); // Never create dust outputs; if we would, just @@ -2666,7 +2674,6 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT { nChangePosInOut = -1; nFeeRet += nChange; - reservekey.ReturnKey(); } else { @@ -2685,7 +2692,6 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT txNew.vout.insert(position, newTxOut); } } else { - reservekey.ReturnKey(); nChangePosInOut = -1; } @@ -2724,7 +2730,10 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT if (coinControl && coinControl->nConfirmTarget > 0) currentConfirmationTarget = coinControl->nConfirmTarget; - CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator, &feeCalc); + // Allow to override the default fee estimate mode over the CoinControl instance + bool conservative_estimate = CalculateEstimateType(coinControl ? coinControl->m_fee_mode : FeeEstimateMode::UNSET, rbf); + + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator, &feeCalc, false /* ignoreGlobalPayTxFee */, conservative_estimate); if (coinControl && coinControl->fOverrideFeeRate) nFeeNeeded = coinControl->nFeeRate.GetFee(nBytes); @@ -2737,16 +2746,30 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT } if (nFeeRet >= nFeeNeeded) { - // Reduce fee to only the needed amount if we have change - // output to increase. This prevents potential overpayment - // in fees if the coins selected to meet nFeeNeeded result - // in a transaction that requires less fee than the prior - // iteration. + // Reduce fee to only the needed amount if possible. This + // prevents potential overpayment in fees if the coins + // selected to meet nFeeNeeded result in a transaction that + // requires less fee than the prior iteration. + // TODO: The case where nSubtractFeeFromAmount > 0 remains // to be addressed because it requires returning the fee to // the payees and not the change output. - // TODO: The case where there is no change output remains - // to be addressed so we avoid creating too small an output. + + // If we have no change and a big enough excess fee, then + // try to construct transaction again only without picking + // new inputs. We now know we only need the smaller fee + // (because of reduced tx size) and so we should add a + // change output. Only try this once. + CAmount fee_needed_for_change = GetMinimumFee(change_prototype_size, currentConfirmationTarget, ::mempool, ::feeEstimator, nullptr, false /* ignoreGlobalPayTxFee */, conservative_estimate); + CAmount minimum_value_for_change = GetDustThreshold(change_prototype_txout, ::dustRelayFee); + CAmount max_excess_fee = fee_needed_for_change + minimum_value_for_change; + if (nFeeRet > nFeeNeeded + max_excess_fee && nChangePosInOut == -1 && nSubtractFeeFromAmount == 0 && pick_new_inputs) { + pick_new_inputs = false; + nFeeRet = nFeeNeeded + fee_needed_for_change; + continue; + } + + // If we have change output already, just increase it if (nFeeRet > nFeeNeeded && nChangePosInOut != -1 && nSubtractFeeFromAmount == 0) { CAmount extraFeePaid = nFeeRet - nFeeNeeded; std::vector<CTxOut>::iterator change_position = txNew.vout.begin()+nChangePosInOut; @@ -2755,6 +2778,12 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT } break; // Done, enough fee included. } + else if (!pick_new_inputs) { + // This shouldn't happen, we should have had enough excess + // fee to pay for the new output and still meet nFeeNeeded + strFailReason = _("Transaction fee and change calculation failed"); + return false; + } // Try to reduce change to include necessary fee if (nChangePosInOut != -1 && nSubtractFeeFromAmount == 0) { @@ -2774,6 +2803,8 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT } } + if (nChangePosInOut == -1) reservekey.ReturnKey(); // Return any reserved key if we don't have change + if (sign) { CTransaction txNewConst(txNew); @@ -2905,13 +2936,13 @@ CAmount CWallet::GetRequiredFee(unsigned int nTxBytes) return std::max(minTxFee.GetFee(nTxBytes), ::minRelayTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation *feeCalc, bool ignoreGlobalPayTxFee) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation *feeCalc, bool ignoreGlobalPayTxFee, bool conservative_estimate) { // payTxFee is the user-set global for desired feerate CAmount nFeeNeeded = payTxFee.GetFee(nTxBytes); // User didn't set: use -txconfirmtarget to estimate... if (nFeeNeeded == 0 || ignoreGlobalPayTxFee) { - nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, feeCalc, pool, true).GetFee(nTxBytes); + nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, feeCalc, pool, conservative_estimate).GetFee(nTxBytes); // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee if (nFeeNeeded == 0) { nFeeNeeded = fallbackFee.GetFee(nTxBytes); @@ -4154,3 +4185,15 @@ bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& { return ::AcceptToMemoryPool(mempool, state, tx, true, NULL, NULL, false, nAbsurdFee); } + +bool CalculateEstimateType(FeeEstimateMode mode, bool opt_in_rbf) { + switch (mode) { + case FeeEstimateMode::UNSET: + return !opt_in_rbf; // Allow for lower fees if RBF is an option + case FeeEstimateMode::CONSERVATIVE: + return true; + case FeeEstimateMode::ECONOMICAL: + return false; + } + return true; +} diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 4f558adc77..e3715cdf37 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -80,6 +80,7 @@ class CTxMemPool; class CBlockPolicyEstimator; class CWalletTx; struct FeeCalculation; +enum class FeeEstimateMode; /** (client) version numbers for particular wallet features */ enum WalletFeature @@ -963,7 +964,7 @@ public: * Estimate the minimum fee considering user set parameters * and the required fee */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation *feeCalc = nullptr, bool ignoreGlobalPayTxFee = false); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation *feeCalc, bool ignoreGlobalPayTxFee, bool conservative_estimate); /** * Return the minimum required fee taking into account the * floating relay fee and user set minimum transaction fee @@ -1211,4 +1212,7 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins } return true; } + +bool CalculateEstimateType(FeeEstimateMode mode, bool opt_in_rbf); + #endif // BITCOIN_WALLET_WALLET_H diff --git a/test/functional/getblocktemplate_proposals.py b/test/functional/getblocktemplate_proposals.py deleted file mode 100755 index fca99c7df5..0000000000 --- a/test/functional/getblocktemplate_proposals.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test block proposals with getblocktemplate.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -from binascii import a2b_hex, b2a_hex -from hashlib import sha256 -from struct import pack - -def b2x(b): - return b2a_hex(b).decode('ascii') - -# NOTE: This does not work for signed numbers (set the high bit) or zero (use b'\0') -def encodeUNum(n): - s = bytearray(b'\1') - while n > 127: - s[0] += 1 - s.append(n % 256) - n //= 256 - s.append(n) - return bytes(s) - -def varlenEncode(n): - if n < 0xfd: - return pack('<B', n) - if n <= 0xffff: - return b'\xfd' + pack('<H', n) - if n <= 0xffffffff: - return b'\xfe' + pack('<L', n) - return b'\xff' + pack('<Q', n) - -def dblsha(b): - return sha256(sha256(b).digest()).digest() - -def genmrklroot(leaflist): - cur = leaflist - while len(cur) > 1: - n = [] - if len(cur) & 1: - cur.append(cur[-1]) - for i in range(0, len(cur), 2): - n.append(dblsha(cur[i] + cur[i+1])) - cur = n - return cur[0] - -def template_to_bytearray(tmpl, txlist): - blkver = pack('<L', tmpl['version']) - mrklroot = genmrklroot(list(dblsha(a) for a in txlist)) - timestamp = pack('<L', tmpl['curtime']) - nonce = b'\0\0\0\0' - blk = blkver + a2b_hex(tmpl['previousblockhash'])[::-1] + mrklroot + timestamp + a2b_hex(tmpl['bits'])[::-1] + nonce - blk += varlenEncode(len(txlist)) - for tx in txlist: - blk += tx - return bytearray(blk) - -def template_to_hex(tmpl, txlist): - return b2x(template_to_bytearray(tmpl, txlist)) - -def assert_template(node, tmpl, txlist, expect): - rsp = node.getblocktemplate({'data':template_to_hex(tmpl, txlist),'mode':'proposal'}) - if rsp != expect: - raise AssertionError('unexpected: %s' % (rsp,)) - -class GetBlockTemplateProposalTest(BitcoinTestFramework): - - def __init__(self): - super().__init__() - self.num_nodes = 2 - self.setup_clean_chain = False - - def run_test(self): - node = self.nodes[0] - node.generate(1) # Mine a block to leave initial block download - tmpl = node.getblocktemplate() - if 'coinbasetxn' not in tmpl: - rawcoinbase = encodeUNum(tmpl['height']) - rawcoinbase += b'\x01-' - hexcoinbase = b2x(rawcoinbase) - hexoutval = b2x(pack('<Q', tmpl['coinbasevalue'])) - tmpl['coinbasetxn'] = {'data': '01000000' + '01' + '0000000000000000000000000000000000000000000000000000000000000000ffffffff' + ('%02x' % (len(rawcoinbase),)) + hexcoinbase + 'fffffffe' + '01' + hexoutval + '00' + '00000000'} - txlist = list(bytearray(a2b_hex(a['data'])) for a in (tmpl['coinbasetxn'],) + tuple(tmpl['transactions'])) - - # Test 0: Capability advertised - assert('proposal' in tmpl['capabilities']) - - # NOTE: This test currently FAILS (regtest mode doesn't enforce block height in coinbase) - ## Test 1: Bad height in coinbase - #txlist[0][4+1+36+1+1] += 1 - #assert_template(node, tmpl, txlist, 'FIXME') - #txlist[0][4+1+36+1+1] -= 1 - - # Test 2: Bad input hash for gen tx - txlist[0][4+1] += 1 - assert_template(node, tmpl, txlist, 'bad-cb-missing') - txlist[0][4+1] -= 1 - - # Test 3: Truncated final tx - lastbyte = txlist[-1].pop() - assert_raises_jsonrpc(-22, "Block decode failed", assert_template, node, tmpl, txlist, 'n/a') - txlist[-1].append(lastbyte) - - # Test 4: Add an invalid tx to the end (duplicate of gen tx) - txlist.append(txlist[0]) - assert_template(node, tmpl, txlist, 'bad-txns-duplicate') - txlist.pop() - - # Test 5: Add an invalid tx to the end (non-duplicate) - txlist.append(bytearray(txlist[0])) - txlist[-1][4+1] = 0xff - assert_template(node, tmpl, txlist, 'bad-txns-inputs-missingorspent') - txlist.pop() - - # Test 6: Future tx lock time - txlist[0][-4:] = b'\xff\xff\xff\xff' - assert_template(node, tmpl, txlist, 'bad-txns-nonfinal') - txlist[0][-4:] = b'\0\0\0\0' - - # Test 7: Bad tx count - txlist.append(b'') - assert_raises_jsonrpc(-22, 'Block decode failed', assert_template, node, tmpl, txlist, 'n/a') - txlist.pop() - - # Test 8: Bad bits - realbits = tmpl['bits'] - tmpl['bits'] = '1c0000ff' # impossible in the real world - assert_template(node, tmpl, txlist, 'bad-diffbits') - tmpl['bits'] = realbits - - # Test 9: Bad merkle root - rawtmpl = template_to_bytearray(tmpl, txlist) - rawtmpl[4+32] = (rawtmpl[4+32] + 1) % 0x100 - rsp = node.getblocktemplate({'data':b2x(rawtmpl),'mode':'proposal'}) - if rsp != 'bad-txnmrklroot': - raise AssertionError('unexpected: %s' % (rsp,)) - - # Test 10: Bad timestamps - realtime = tmpl['curtime'] - tmpl['curtime'] = 0x7fffffff - assert_template(node, tmpl, txlist, 'time-too-new') - tmpl['curtime'] = 0 - assert_template(node, tmpl, txlist, 'time-too-old') - tmpl['curtime'] = realtime - - # Test 11: Valid block - assert_template(node, tmpl, txlist, None) - - # Test 12: Orphan block - tmpl['previousblockhash'] = 'ff00' * 16 - assert_template(node, tmpl, txlist, 'inconclusive-not-best-prevblk') - -if __name__ == '__main__': - GetBlockTemplateProposalTest().main() diff --git a/test/functional/mining.py b/test/functional/mining.py new file mode 100755 index 0000000000..dbd4e29eca --- /dev/null +++ b/test/functional/mining.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test mining RPCs + +- getblocktemplate proposal mode +- submitblock""" + +from binascii import b2a_hex +import copy + +from test_framework.blocktools import create_coinbase +from test_framework.test_framework import BitcoinTestFramework +from test_framework.mininode import CBlock +from test_framework.util import * + +def b2x(b): + return b2a_hex(b).decode('ascii') + +def assert_template(node, block, expect, rehash=True): + if rehash: + block.hashMerkleRoot = block.calc_merkle_root() + rsp = node.getblocktemplate({'data': b2x(block.serialize()), 'mode': 'proposal'}) + assert_equal(rsp, expect) + +class MiningTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False + + def run_test(self): + node = self.nodes[0] + # Mine a block to leave initial block download + node.generate(1) + tmpl = node.getblocktemplate() + self.log.info("getblocktemplate: Test capability advertised") + assert 'proposal' in tmpl['capabilities'] + assert 'coinbasetxn' not in tmpl + + coinbase_tx = create_coinbase(height=int(tmpl["height"]) + 1) + # sequence numbers must not be max for nLockTime to have effect + coinbase_tx.vin[0].nSequence = 2 ** 32 - 2 + coinbase_tx.rehash() + + block = CBlock() + block.nVersion = tmpl["version"] + block.hashPrevBlock = int(tmpl["previousblockhash"], 16) + block.nTime = tmpl["curtime"] + block.nBits = int(tmpl["bits"], 16) + block.nNonce = 0 + block.vtx = [coinbase_tx] + + self.log.info("getblocktemplate: Test valid block") + assert_template(node, block, None) + + self.log.info("submitblock: Test block decode failure") + assert_raises_jsonrpc(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) + + self.log.info("getblocktemplate: Test bad input hash for coinbase transaction") + bad_block = copy.deepcopy(block) + bad_block.vtx[0].vin[0].prevout.hash += 1 + bad_block.vtx[0].rehash() + assert_template(node, bad_block, 'bad-cb-missing') + + self.log.info("submitblock: Test invalid coinbase transaction") + assert_raises_jsonrpc(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) + + self.log.info("getblocktemplate: Test truncated final transaction") + assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'}) + + self.log.info("getblocktemplate: Test duplicate transaction") + bad_block = copy.deepcopy(block) + bad_block.vtx.append(bad_block.vtx[0]) + assert_template(node, bad_block, 'bad-txns-duplicate') + + self.log.info("getblocktemplate: Test invalid transaction") + bad_block = copy.deepcopy(block) + bad_tx = copy.deepcopy(bad_block.vtx[0]) + bad_tx.vin[0].prevout.hash = 255 + bad_tx.rehash() + bad_block.vtx.append(bad_tx) + assert_template(node, bad_block, 'bad-txns-inputs-missingorspent') + + self.log.info("getblocktemplate: Test nonfinal transaction") + bad_block = copy.deepcopy(block) + bad_block.vtx[0].nLockTime = 2 ** 32 - 1 + bad_block.vtx[0].rehash() + assert_template(node, bad_block, 'bad-txns-nonfinal') + + self.log.info("getblocktemplate: Test bad tx count") + # The tx count is immediately after the block header + TX_COUNT_OFFSET = 80 + bad_block_sn = bytearray(block.serialize()) + assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1) + bad_block_sn[TX_COUNT_OFFSET] += 1 + assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'}) + + self.log.info("getblocktemplate: Test bad bits") + bad_block = copy.deepcopy(block) + bad_block.nBits = 469762303 # impossible in the real world + assert_template(node, bad_block, 'bad-diffbits') + + self.log.info("getblocktemplate: Test bad merkle root") + bad_block = copy.deepcopy(block) + bad_block.hashMerkleRoot += 1 + assert_template(node, bad_block, 'bad-txnmrklroot', False) + + self.log.info("getblocktemplate: Test bad timestamps") + bad_block = copy.deepcopy(block) + bad_block.nTime = 2 ** 31 - 1 + assert_template(node, bad_block, 'time-too-new') + bad_block.nTime = 0 + assert_template(node, bad_block, 'time-too-old') + + self.log.info("getblocktemplate: Test not best block") + bad_block = copy.deepcopy(block) + bad_block.hashPrevBlock = 123 + assert_template(node, bad_block, 'inconclusive-not-best-prevblk') + +if __name__ == '__main__': + MiningTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 54f625514b..b7bc6e841b 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -108,6 +108,7 @@ BASE_SCRIPTS= [ 'signmessages.py', 'nulldummy.py', 'import-rescan.py', + 'mining.py', 'bumpfee.py', 'rpcnamedargs.py', 'listsinceblock.py', @@ -141,7 +142,6 @@ EXTENDED_SCRIPTS = [ 'bipdersig-p2p.py', 'bipdersig.py', 'example_test.py', - 'getblocktemplate_proposals.py', 'txn_doublespend.py', 'txn_clone.py --mineblock', 'forknotify.py', |