diff options
| -rw-r--r-- | .travis.yml | 3 | ||||
| -rw-r--r-- | contrib/linearize/README.md | 10 | ||||
| -rw-r--r-- | contrib/linearize/example-linearize.cfg | 13 | ||||
| -rwxr-xr-x | contrib/linearize/linearize-data.py | 36 | ||||
| -rwxr-xr-x | contrib/verify-commits/verify-commits.sh | 3 | ||||
| -rw-r--r-- | src/hash.cpp | 2 | ||||
| -rw-r--r-- | src/init.cpp | 5 | ||||
| -rw-r--r-- | src/rpc/mining.cpp | 4 | ||||
| -rw-r--r-- | src/test/versionbits_tests.cpp | 2 | ||||
| -rw-r--r-- | src/util.cpp | 8 | ||||
| -rw-r--r-- | src/wallet/rpcwallet.cpp | 55 | ||||
| -rw-r--r-- | src/wallet/wallet.cpp | 18 | ||||
| -rw-r--r-- | src/wallet/wallet.h | 27 |
13 files changed, 138 insertions, 48 deletions
diff --git a/.travis.yml b/.travis.yml index 4087a854b4..ccd2490925 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,6 +51,9 @@ before_script: - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS script: + - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi + - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then git fetch --unshallow; fi + - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then contrib/verify-commits/verify-commits.sh; fi - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index adc9a559cc..0971e7816b 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -32,8 +32,11 @@ Required configuration file settings: * `output`: Output directory for linearized `blocks/blkNNNNN.dat` output. Optional config file setting for linearize-data: -* `file_timestamp`: Set each file's last-modified time to that of the most -recent block in that file. +* `debug_output`: Some printouts may not always be desired. If true, such output +will be printed. +* `file_timestamp`: Set each file's last-accessed and last-modified times, +respectively, to the current time and to the timestamp of the most recent block +written to the script's blockchain. * `genesis`: The hash of the genesis block in the blockchain. * `input`: bitcoind blocks/ directory containing blkNNNNN.dat * `hashlist`: text file containing list of block hashes created by @@ -41,6 +44,9 @@ linearize-hashes.py. * `max_out_sz`: Maximum size for files created by the `output_file` option. (Default: `1000*1000*1000 bytes`) * `netmagic`: Network magic number. +* `out_of_order_cache_sz`: If out-of-order blocks are being read, the block can +be written to a cache so that the blockchain doesn't have to be seeked again. +This option specifies the cache size. (Default: `100*1000*1000 bytes`) * `rev_hash_bytes`: If true, the block hash list written by linearize-hashes.py will be byte-reversed when read by linearize-data.py. See the linearize-hashes entry for more information. diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 69f0e9247a..2cc910edfe 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -1,4 +1,3 @@ - # bitcoind RPC settings (linearize-hashes) rpcuser=someuser rpcpassword=somepassword @@ -21,6 +20,9 @@ input=/home/example/.bitcoin/blocks #genesis=000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943 #input=/home/example/.bitcoin/testnet3/blocks +# "output" option causes blockchain files to be written to the given location, +# with "output_file" ignored. If not used, "output_file" is used instead. +# output=/home/example/blockchain_directory output_file=/home/example/Downloads/bootstrap.dat hashlist=hashlist.txt @@ -29,3 +31,12 @@ out_of_order_cache_sz = 100000000 # Do we want the reverse the hash bytes coming from getblockhash? rev_hash_bytes = False + +# On a new month, do we want to set the access and modify times of the new +# blockchain file? +file_timestamp = 0 +# Do we want to split the blockchain files given a new month or specific height? +split_timestamp = 0 + +# Do we want debug printouts? +debug_output = False diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 3fdec134b8..afcec2b60a 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -134,7 +134,7 @@ class BlockDataCopier: if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): self.outF.close() if self.setFileTime: - os.utime(outFname, (int(time.time()), highTS)) + os.utime(self.outFname, (int(time.time()), self.highTS)) self.outF = None self.outFname = None self.outFn = self.outFn + 1 @@ -142,12 +142,12 @@ class BlockDataCopier: (blkDate, blkTS) = get_blk_dt(blk_hdr) if self.timestampSplit and (blkDate > self.lastDate): - print("New month " + blkDate.strftime("%Y-%m") + " @ " + hash_str) - lastDate = blkDate - if outF: - outF.close() - if setFileTime: - os.utime(outFname, (int(time.time()), highTS)) + print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) + self.lastDate = blkDate + if self.outF: + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) self.outF = None self.outFname = None self.outFn = self.outFn + 1 @@ -155,11 +155,11 @@ class BlockDataCopier: if not self.outF: if self.fileOutput: - outFname = self.settings['output_file'] + self.outFname = self.settings['output_file'] else: - outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) - print("Output file " + outFname) - self.outF = open(outFname, "wb") + self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) + print("Output file " + self.outFname) + self.outF = open(self.outFname, "wb") self.outF.write(inhdr) self.outF.write(blk_hdr) @@ -223,13 +223,16 @@ class BlockDataCopier: blk_hdr = self.inF.read(80) inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen) - hash_str = calc_hash_str(blk_hdr) - if not hash_str in blkmap: - print("Skipping unknown block " + hash_str) + self.hash_str = calc_hash_str(blk_hdr) + if not self.hash_str in blkmap: + # Because blocks can be written to files out-of-order as of 0.10, the script + # may encounter blocks it doesn't know about. Treat as debug output. + if settings['debug_output'] == 'true': + print("Skipping unknown block " + self.hash_str) self.inF.seek(inLen, os.SEEK_CUR) continue - blkHeight = self.blkmap[hash_str] + blkHeight = self.blkmap[self.hash_str] self.blkCountIn += 1 if self.blkCountOut == blkHeight: @@ -295,12 +298,15 @@ if __name__ == '__main__': settings['max_out_sz'] = 1000 * 1000 * 1000 if 'out_of_order_cache_sz' not in settings: settings['out_of_order_cache_sz'] = 100 * 1000 * 1000 + if 'debug_output' not in settings: + settings['debug_output'] = 'false' settings['max_out_sz'] = int(settings['max_out_sz']) settings['split_timestamp'] = int(settings['split_timestamp']) settings['file_timestamp'] = int(settings['file_timestamp']) settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8')) settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) + settings['debug_output'] = settings['debug_output'].lower() if 'output_file' not in settings and 'output' not in settings: print("Missing output file / directory") diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh index cfe4f11a0b..b2cebdf1a0 100755 --- a/contrib/verify-commits/verify-commits.sh +++ b/contrib/verify-commits/verify-commits.sh @@ -28,9 +28,10 @@ IS_SIGNED () { local PARENTS PARENTS=$(git show -s --format=format:%P $1) for PARENT in $PARENTS; do - if IS_SIGNED $PARENT > /dev/null; then + if IS_SIGNED $PARENT; then return 0; fi + break done if ! "$HAVE_FAILED"; then echo "No parent of $1 was signed with a trusted key!" > /dev/stderr diff --git a/src/hash.cpp b/src/hash.cpp index a399ca9252..a14a2386a2 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -57,7 +57,7 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char k1 = ROTL32(k1, 15); k1 *= c2; h1 ^= k1; - }; + } } //---------- diff --git a/src/init.cpp b/src/init.cpp index d7d60b0fbc..7c108ac4a6 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1146,8 +1146,11 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) #ifndef WIN32 CreatePidFile(GetPidFile(), getpid()); #endif - if (GetBoolArg("-shrinkdebugfile", !fDebug)) + if (GetBoolArg("-shrinkdebugfile", !fDebug)) { + // Do this first since it both loads a bunch of debug.log into memory, + // and because this needs to happen before any other debug.log printing ShrinkDebugFile(); + } if (fPrintToDebugLog) OpenDebugLog(); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index e46f55a8aa..c594daca0d 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -697,7 +697,7 @@ public: bool found; CValidationState state; - submitblock_StateCatcher(const uint256 &hashIn) : hash(hashIn), found(false), state() {}; + submitblock_StateCatcher(const uint256 &hashIn) : hash(hashIn), found(false), state() {} protected: virtual void BlockChecked(const CBlock& block, const CValidationState& stateIn) { @@ -705,7 +705,7 @@ protected: return; found = true; state = stateIn; - }; + } }; UniValue submitblock(const JSONRPCRequest& request) diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index bae0eff7e5..e2b5573abd 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -292,7 +292,7 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) blocksToMine--; nTime += 600; nHeight += 1; - }; + } nTime = nTimeout; // FAILED is only triggered at the end of a period, so CBV should be setting diff --git a/src/util.cpp b/src/util.cpp index 08ee6b8b87..ba157625d8 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -723,13 +723,17 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { void ShrinkDebugFile() { + // Amount of debug.log to save at end when shrinking (must fit in memory) + constexpr size_t RECENT_DEBUG_HISTORY_SIZE = 10 * 1000000; // Scroll debug.log if it's getting too big boost::filesystem::path pathLog = GetDataDir() / "debug.log"; FILE* file = fopen(pathLog.string().c_str(), "r"); - if (file && boost::filesystem::file_size(pathLog) > 10 * 1000000) + // If debug.log file is more than 10% bigger the RECENT_DEBUG_HISTORY_SIZE + // trim it down by saving only the last RECENT_DEBUG_HISTORY_SIZE bytes + if (file && boost::filesystem::file_size(pathLog) > 11 * (RECENT_DEBUG_HISTORY_SIZE / 10)) { // Restart the file with some of the end - std::vector <char> vch(200000,0); + std::vector<char> vch(RECENT_DEBUG_HISTORY_SIZE, 0); fseek(file, -((long)vch.size()), SEEK_END); int nBytes = fread(vch.data(), 1, vch.size(), file); fclose(file); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index bfc738afad..36753d1116 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2416,7 +2416,7 @@ UniValue listunspent(const JSONRPCRequest& request) " \"address\" : \"address\", (string) the bitcoin address\n" " \"account\" : \"account\", (string) DEPRECATED. The associated account, or \"\" for the default account\n" " \"scriptPubKey\" : \"key\", (string) the script key\n" - " \"amount\" : x.xxx, (numeric) the transaction amount in " + CURRENCY_UNIT + "\n" + " \"amount\" : x.xxx, (numeric) the transaction output amount in " + CURRENCY_UNIT + "\n" " \"confirmations\" : n, (numeric) The number of confirmations\n" " \"redeemScript\" : n (string) The redeemScript if scriptPubKey is P2SH\n" " \"spendable\" : xxx, (bool) Whether we have the private keys to spend this output\n" @@ -2664,6 +2664,33 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) return result; } +// Calculate the size of the transaction assuming all signatures are max size +// Use DummySignatureCreator, which inserts 72 byte signatures everywhere. +// TODO: re-use this in CWallet::CreateTransaction (right now +// CreateTransaction uses the constructed dummy-signed tx to do a priority +// calculation, but we should be able to refactor after priority is removed). +// NOTE: this requires that all inputs must be in mapWallet (eg the tx should +// be IsAllFromMe). +int64_t CalculateMaximumSignedTxSize(const CTransaction &tx) +{ + CMutableTransaction txNew(tx); + std::vector<pair<CWalletTx *, unsigned int>> vCoins; + // Look up the inputs. We should have already checked that this transaction + // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our + // wallet, with a valid index into the vout array. + for (auto& input : tx.vin) { + const auto mi = pwalletMain->mapWallet.find(input.prevout.hash); + assert(mi != pwalletMain->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); + vCoins.emplace_back(make_pair(&(mi->second), input.prevout.n)); + } + if (!pwalletMain->DummySignTx(txNew, vCoins)) { + // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) + // implies that we can sign for every input. + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction contains inputs that cannot be signed"); + } + return GetVirtualTransactionSize(txNew); +} + UniValue bumpfee(const JSONRPCRequest& request) { if (!EnsureWalletIsAvailable(request.fHelp)) { @@ -2706,6 +2733,7 @@ UniValue bumpfee(const JSONRPCRequest& request) " \"txid\": \"value\", (string) The id of the new transaction\n" " \"origfee\": n, (numeric) Fee of the replaced transaction\n" " \"fee\": n, (numeric) Fee of the new transaction\n" + " \"errors\": [ str... ] (json array of strings) Errors encountered during processing (may be empty)\n" "}\n" "\nExamples:\n" "\nBump the fee, get the new transaction\'s txid\n" + @@ -2769,9 +2797,9 @@ UniValue bumpfee(const JSONRPCRequest& request) throw JSONRPCError(RPC_MISC_ERROR, "Transaction does not have a change output"); } - // signature sizes can vary by a byte, so add 1 for each input when calculating the new fee + // Calculate the expected size of the new transaction. int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); - const int64_t maxNewTxSize = txSize + wtx.tx->vin.size(); + const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx); // optional parameters bool specifiedConfirmTarget = false; @@ -2845,8 +2873,12 @@ UniValue bumpfee(const JSONRPCRequest& request) nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); // New fee rate must be at least old rate + minimum incremental relay rate - if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + walletIncrementalRelayFee.GetFeePerK()) { - nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + walletIncrementalRelayFee.GetFeePerK()); + // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized + // in that unit (fee per kb). + // However, nOldFeeRate is a calculated value from the tx fee/size, so + // add 1 satoshi to the result, because it may have been rounded down. + if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { + nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); nNewFee = nNewFeeRate.GetFee(maxNewTxSize); } } @@ -2914,23 +2946,32 @@ UniValue bumpfee(const JSONRPCRequest& request) CWalletTx wtxBumped(pwalletMain, MakeTransactionRef(std::move(tx))); wtxBumped.mapValue["replaces_txid"] = hash.ToString(); CValidationState state; - if (!pwalletMain->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state) || !state.IsValid()) { + if (!pwalletMain->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { + // NOTE: CommitTransaction never returns false, so this should never happen. throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); } + UniValue vErrors(UniValue::VARR); + if (state.IsInvalid()) { + // This can happen if the mempool rejected the transaction. Report + // what happened in the "errors" response. + vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); + } + // mark the original tx as bumped if (!pwalletMain->MarkReplaced(wtx.GetHash(), wtxBumped.GetHash())) { // TODO: see if JSON-RPC has a standard way of returning a response // along with an exception. It would be good to return information about // wtxBumped to the caller even if marking the original transaction // replaced does not succeed for some reason. - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); + vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); } UniValue result(UniValue::VOBJ); result.push_back(Pair("txid", wtxBumped.GetHash().GetHex())); result.push_back(Pair("origfee", ValueFromAmount(nOldFee))); result.push_back(Pair("fee", ValueFromAmount(nNewFee))); + result.push_back(Pair("errors", vErrors)); return result; } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 9a5f35b6e3..a7b8022bd9 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2583,21 +2583,9 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt std::numeric_limits<unsigned int>::max() - (fWalletRbf ? 2 : 1))); // Fill in dummy signatures for fee calculation. - int nIn = 0; - for (const auto& coin : setCoins) - { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; - SignatureData sigdata; - - if (!ProduceSignature(DummySignatureCreator(this), scriptPubKey, sigdata)) - { - strFailReason = _("Signing transaction failed"); - return false; - } else { - UpdateTransaction(txNew, nIn, sigdata); - } - - nIn++; + if (!DummySignTx(txNew, setCoins)) { + strFailReason = _("Signing transaction failed"); + return false; } unsigned int nBytes = GetVirtualTransactionSize(txNew); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 200ec0cba2..1de04ae16a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -13,6 +13,7 @@ #include "utilstrencodings.h" #include "validationinterface.h" #include "script/ismine.h" +#include "script/sign.h" #include "wallet/crypter.h" #include "wallet/walletdb.h" #include "wallet/rpcwallet.h" @@ -796,6 +797,8 @@ public: void ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries); bool AddAccountingEntry(const CAccountingEntry&); bool AddAccountingEntry(const CAccountingEntry&, CWalletDB *pwalletdb); + template <typename ContainerType> + bool DummySignTx(CMutableTransaction &txNew, const ContainerType &coins); static CFeeRate minTxFee; static CFeeRate fallbackFee; @@ -1028,4 +1031,28 @@ public: } }; +// Helper for producing a bunch of max-sized low-S signatures (eg 72 bytes) +// ContainerType is meant to hold pair<CWalletTx *, int>, and be iterable +// so that each entry corresponds to each vIn, in order. +template <typename ContainerType> +bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) +{ + // Fill in dummy signatures for fee calculation. + int nIn = 0; + for (const auto& coin : coins) + { + const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + SignatureData sigdata; + + if (!ProduceSignature(DummySignatureCreator(this), scriptPubKey, sigdata)) + { + return false; + } else { + UpdateTransaction(txNew, nIn, sigdata); + } + + nIn++; + } + return true; +} #endif // BITCOIN_WALLET_WALLET_H |